2024-12-09 10:40:21,959 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-12-09 10:40:21,974 main DEBUG Took 0.012355 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-09 10:40:21,974 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-09 10:40:21,975 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-09 10:40:21,976 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-09 10:40:21,977 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 10:40:21,986 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-09 10:40:22,003 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 10:40:22,004 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 10:40:22,005 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 10:40:22,005 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 10:40:22,006 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 10:40:22,007 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 10:40:22,008 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 10:40:22,008 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 10:40:22,011 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 10:40:22,011 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 10:40:22,018 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 10:40:22,018 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 10:40:22,019 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 10:40:22,020 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 10:40:22,021 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 10:40:22,021 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 10:40:22,022 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 10:40:22,022 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 10:40:22,023 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 10:40:22,023 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 10:40:22,024 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 10:40:22,024 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 10:40:22,025 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 10:40:22,025 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 10:40:22,026 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 10:40:22,026 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-09 10:40:22,028 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 10:40:22,030 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-09 10:40:22,032 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-09 10:40:22,033 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-09 10:40:22,035 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-09 10:40:22,035 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-09 10:40:22,047 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-09 10:40:22,051 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-09 10:40:22,053 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-09 10:40:22,053 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-09 10:40:22,054 main DEBUG createAppenders(={Console}) 2024-12-09 10:40:22,055 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-12-09 10:40:22,056 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-12-09 10:40:22,056 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-12-09 10:40:22,057 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-09 10:40:22,057 main DEBUG OutputStream closed 2024-12-09 10:40:22,057 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-09 10:40:22,058 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-09 10:40:22,058 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-12-09 10:40:22,178 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-09 10:40:22,181 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-09 10:40:22,182 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-09 10:40:22,184 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-09 10:40:22,185 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-09 10:40:22,185 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-09 10:40:22,186 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-09 10:40:22,186 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-09 10:40:22,186 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-09 10:40:22,187 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-09 10:40:22,187 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-09 10:40:22,188 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-09 10:40:22,188 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-09 10:40:22,189 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-09 10:40:22,189 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-09 10:40:22,190 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-09 10:40:22,190 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-09 10:40:22,191 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-09 10:40:22,194 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-09 10:40:22,194 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-12-09 10:40:22,195 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-09 10:40:22,196 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-12-09T10:40:22,609 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/376c0fb7-94bc-49c9-3c3c-f09e60d8d2eb 2024-12-09 10:40:22,613 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-09 10:40:22,613 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-09T10:40:22,630 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-12-09T10:40:22,680 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-09T10:40:22,698 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/376c0fb7-94bc-49c9-3c3c-f09e60d8d2eb/cluster_8fa5eac4-e45c-afe9-0851-24d1455445f8, deleteOnExit=true 2024-12-09T10:40:22,699 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-09T10:40:22,700 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/376c0fb7-94bc-49c9-3c3c-f09e60d8d2eb/test.cache.data in system properties and HBase conf 2024-12-09T10:40:22,701 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/376c0fb7-94bc-49c9-3c3c-f09e60d8d2eb/hadoop.tmp.dir in system properties and HBase conf 2024-12-09T10:40:22,701 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/376c0fb7-94bc-49c9-3c3c-f09e60d8d2eb/hadoop.log.dir in system properties and HBase conf 2024-12-09T10:40:22,702 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/376c0fb7-94bc-49c9-3c3c-f09e60d8d2eb/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-09T10:40:22,703 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/376c0fb7-94bc-49c9-3c3c-f09e60d8d2eb/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-09T10:40:22,703 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-09T10:40:22,812 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-09T10:40:22,940 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-09T10:40:22,946 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/376c0fb7-94bc-49c9-3c3c-f09e60d8d2eb/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-09T10:40:22,947 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/376c0fb7-94bc-49c9-3c3c-f09e60d8d2eb/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-09T10:40:22,948 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/376c0fb7-94bc-49c9-3c3c-f09e60d8d2eb/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-09T10:40:22,948 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/376c0fb7-94bc-49c9-3c3c-f09e60d8d2eb/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-09T10:40:22,949 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/376c0fb7-94bc-49c9-3c3c-f09e60d8d2eb/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-09T10:40:22,949 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/376c0fb7-94bc-49c9-3c3c-f09e60d8d2eb/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-09T10:40:22,950 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/376c0fb7-94bc-49c9-3c3c-f09e60d8d2eb/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-09T10:40:22,950 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/376c0fb7-94bc-49c9-3c3c-f09e60d8d2eb/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-09T10:40:22,951 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/376c0fb7-94bc-49c9-3c3c-f09e60d8d2eb/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-09T10:40:22,951 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/376c0fb7-94bc-49c9-3c3c-f09e60d8d2eb/nfs.dump.dir in system properties and HBase conf 2024-12-09T10:40:22,952 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/376c0fb7-94bc-49c9-3c3c-f09e60d8d2eb/java.io.tmpdir in system properties and HBase conf 2024-12-09T10:40:22,952 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/376c0fb7-94bc-49c9-3c3c-f09e60d8d2eb/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-09T10:40:22,953 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/376c0fb7-94bc-49c9-3c3c-f09e60d8d2eb/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-09T10:40:22,954 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/376c0fb7-94bc-49c9-3c3c-f09e60d8d2eb/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-09T10:40:23,909 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-09T10:40:24,004 INFO [Time-limited test {}] log.Log(170): Logging initialized @2950ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-09T10:40:24,102 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-09T10:40:24,194 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-09T10:40:24,218 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-09T10:40:24,218 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-09T10:40:24,220 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-09T10:40:24,236 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-09T10:40:24,239 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4516de21{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/376c0fb7-94bc-49c9-3c3c-f09e60d8d2eb/hadoop.log.dir/,AVAILABLE} 2024-12-09T10:40:24,240 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6efc0ba4{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-09T10:40:24,477 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@55a6f9c7{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/376c0fb7-94bc-49c9-3c3c-f09e60d8d2eb/java.io.tmpdir/jetty-localhost-38085-hadoop-hdfs-3_4_1-tests_jar-_-any-11190577317160906034/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-09T10:40:24,489 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@40742525{HTTP/1.1, (http/1.1)}{localhost:38085} 2024-12-09T10:40:24,490 INFO [Time-limited test {}] server.Server(415): Started @3437ms 2024-12-09T10:40:24,915 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-09T10:40:24,925 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-09T10:40:24,927 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-09T10:40:24,927 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-09T10:40:24,927 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-09T10:40:24,929 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@603a60a2{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/376c0fb7-94bc-49c9-3c3c-f09e60d8d2eb/hadoop.log.dir/,AVAILABLE} 2024-12-09T10:40:24,929 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@67a8203b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-09T10:40:25,064 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3615128e{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/376c0fb7-94bc-49c9-3c3c-f09e60d8d2eb/java.io.tmpdir/jetty-localhost-44605-hadoop-hdfs-3_4_1-tests_jar-_-any-13832314693309393629/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-09T10:40:25,065 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@580c1d20{HTTP/1.1, (http/1.1)}{localhost:44605} 2024-12-09T10:40:25,065 INFO [Time-limited test {}] server.Server(415): Started @4012ms 2024-12-09T10:40:25,123 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-09T10:40:25,744 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/376c0fb7-94bc-49c9-3c3c-f09e60d8d2eb/cluster_8fa5eac4-e45c-afe9-0851-24d1455445f8/data/data2/current/BP-1165291760-172.17.0.3-1733740823632/current, will proceed with Du for space computation calculation, 2024-12-09T10:40:25,744 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/376c0fb7-94bc-49c9-3c3c-f09e60d8d2eb/cluster_8fa5eac4-e45c-afe9-0851-24d1455445f8/data/data1/current/BP-1165291760-172.17.0.3-1733740823632/current, will proceed with Du for space computation calculation, 2024-12-09T10:40:25,841 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-09T10:40:25,906 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf0834a75ab264134 with lease ID 0x288bf0144d98ca88: Processing first storage report for DS-334b36eb-fd29-4148-a54e-e73877f2c3da from datanode DatanodeRegistration(127.0.0.1:34899, datanodeUuid=db470eaf-d669-4d74-9769-0b3e76661fc5, infoPort=39659, infoSecurePort=0, ipcPort=44015, storageInfo=lv=-57;cid=testClusterID;nsid=200555503;c=1733740823632) 2024-12-09T10:40:25,910 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf0834a75ab264134 with lease ID 0x288bf0144d98ca88: from storage DS-334b36eb-fd29-4148-a54e-e73877f2c3da node DatanodeRegistration(127.0.0.1:34899, datanodeUuid=db470eaf-d669-4d74-9769-0b3e76661fc5, infoPort=39659, infoSecurePort=0, ipcPort=44015, storageInfo=lv=-57;cid=testClusterID;nsid=200555503;c=1733740823632), blocks: 0, hasStaleStorage: true, processing time: 3 msecs, invalidatedBlocks: 0 2024-12-09T10:40:25,910 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf0834a75ab264134 with lease ID 0x288bf0144d98ca88: Processing first storage report for DS-606a9982-2b6c-49bf-a0a3-c73e57f8372e from datanode DatanodeRegistration(127.0.0.1:34899, datanodeUuid=db470eaf-d669-4d74-9769-0b3e76661fc5, infoPort=39659, infoSecurePort=0, ipcPort=44015, storageInfo=lv=-57;cid=testClusterID;nsid=200555503;c=1733740823632) 2024-12-09T10:40:25,910 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf0834a75ab264134 with lease ID 0x288bf0144d98ca88: from storage DS-606a9982-2b6c-49bf-a0a3-c73e57f8372e node DatanodeRegistration(127.0.0.1:34899, datanodeUuid=db470eaf-d669-4d74-9769-0b3e76661fc5, infoPort=39659, infoSecurePort=0, ipcPort=44015, storageInfo=lv=-57;cid=testClusterID;nsid=200555503;c=1733740823632), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-09T10:40:25,940 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/376c0fb7-94bc-49c9-3c3c-f09e60d8d2eb 2024-12-09T10:40:26,035 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/376c0fb7-94bc-49c9-3c3c-f09e60d8d2eb/cluster_8fa5eac4-e45c-afe9-0851-24d1455445f8/zookeeper_0, clientPort=50851, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/376c0fb7-94bc-49c9-3c3c-f09e60d8d2eb/cluster_8fa5eac4-e45c-afe9-0851-24d1455445f8/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/376c0fb7-94bc-49c9-3c3c-f09e60d8d2eb/cluster_8fa5eac4-e45c-afe9-0851-24d1455445f8/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-09T10:40:26,056 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=50851 2024-12-09T10:40:26,072 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T10:40:26,078 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T10:40:26,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741825_1001 (size=7) 2024-12-09T10:40:26,771 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160 with version=8 2024-12-09T10:40:26,771 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/hbase-staging 2024-12-09T10:40:26,866 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-09T10:40:27,107 INFO [Time-limited test {}] client.ConnectionUtils(128): master/7ea43b39fc24:0 server-side Connection retries=45 2024-12-09T10:40:27,120 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T10:40:27,120 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-09T10:40:27,125 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-09T10:40:27,125 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T10:40:27,125 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-09T10:40:27,282 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-09T10:40:27,347 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-09T10:40:27,360 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-09T10:40:27,365 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-09T10:40:27,402 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 1543 (auto-detected) 2024-12-09T10:40:27,404 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-12-09T10:40:27,432 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:35095 2024-12-09T10:40:27,457 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:35095 connecting to ZooKeeper ensemble=127.0.0.1:50851 2024-12-09T10:40:27,491 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:350950x0, quorum=127.0.0.1:50851, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:40:27,494 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:35095-0x100bd55aca10000 connected 2024-12-09T10:40:27,530 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T10:40:27,532 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T10:40:27,550 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35095-0x100bd55aca10000, quorum=127.0.0.1:50851, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T10:40:27,554 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160, hbase.cluster.distributed=false 2024-12-09T10:40:27,581 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35095-0x100bd55aca10000, quorum=127.0.0.1:50851, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-09T10:40:27,590 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35095 2024-12-09T10:40:27,590 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35095 2024-12-09T10:40:27,591 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35095 2024-12-09T10:40:27,592 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35095 2024-12-09T10:40:27,592 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35095 2024-12-09T10:40:27,720 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/7ea43b39fc24:0 server-side Connection retries=45 2024-12-09T10:40:27,723 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T10:40:27,724 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-09T10:40:27,724 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-09T10:40:27,724 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T10:40:27,724 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-09T10:40:27,728 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-09T10:40:27,731 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-09T10:40:27,733 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:36101 2024-12-09T10:40:27,736 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:36101 connecting to ZooKeeper ensemble=127.0.0.1:50851 2024-12-09T10:40:27,737 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T10:40:27,740 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T10:40:27,746 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:361010x0, quorum=127.0.0.1:50851, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T10:40:27,747 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36101-0x100bd55aca10001 connected 2024-12-09T10:40:27,747 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36101-0x100bd55aca10001, quorum=127.0.0.1:50851, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T10:40:27,753 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-09T10:40:27,775 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-09T10:40:27,778 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36101-0x100bd55aca10001, quorum=127.0.0.1:50851, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-09T10:40:27,783 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36101-0x100bd55aca10001, quorum=127.0.0.1:50851, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-09T10:40:27,785 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36101 2024-12-09T10:40:27,787 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36101 2024-12-09T10:40:27,787 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36101 2024-12-09T10:40:27,788 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36101 2024-12-09T10:40:27,789 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36101 2024-12-09T10:40:27,810 DEBUG [M:0;7ea43b39fc24:35095 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;7ea43b39fc24:35095 2024-12-09T10:40:27,811 INFO [master/7ea43b39fc24:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/7ea43b39fc24,35095,1733740826918 2024-12-09T10:40:27,820 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36101-0x100bd55aca10001, quorum=127.0.0.1:50851, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T10:40:27,820 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35095-0x100bd55aca10000, quorum=127.0.0.1:50851, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T10:40:27,822 DEBUG [master/7ea43b39fc24:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35095-0x100bd55aca10000, quorum=127.0.0.1:50851, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/7ea43b39fc24,35095,1733740826918 2024-12-09T10:40:27,849 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35095-0x100bd55aca10000, quorum=127.0.0.1:50851, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T10:40:27,849 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36101-0x100bd55aca10001, quorum=127.0.0.1:50851, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-09T10:40:27,850 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36101-0x100bd55aca10001, quorum=127.0.0.1:50851, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T10:40:27,851 DEBUG [master/7ea43b39fc24:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35095-0x100bd55aca10000, quorum=127.0.0.1:50851, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-09T10:40:27,852 INFO [master/7ea43b39fc24:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/7ea43b39fc24,35095,1733740826918 from backup master directory 2024-12-09T10:40:27,855 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35095-0x100bd55aca10000, quorum=127.0.0.1:50851, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/7ea43b39fc24,35095,1733740826918 2024-12-09T10:40:27,855 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36101-0x100bd55aca10001, quorum=127.0.0.1:50851, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T10:40:27,855 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35095-0x100bd55aca10000, quorum=127.0.0.1:50851, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T10:40:27,856 WARN [master/7ea43b39fc24:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-09T10:40:27,856 INFO [master/7ea43b39fc24:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=7ea43b39fc24,35095,1733740826918 2024-12-09T10:40:27,858 INFO [master/7ea43b39fc24:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-09T10:40:27,860 INFO [master/7ea43b39fc24:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-09T10:40:27,930 DEBUG [master/7ea43b39fc24:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/hbase.id] with ID: 0945b5ae-b46b-4b8f-af48-56b0bb92df09 2024-12-09T10:40:27,930 DEBUG [master/7ea43b39fc24:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/.tmp/hbase.id 2024-12-09T10:40:27,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741826_1002 (size=42) 2024-12-09T10:40:28,344 DEBUG [master/7ea43b39fc24:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/.tmp/hbase.id]:[hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/hbase.id] 2024-12-09T10:40:28,396 INFO [master/7ea43b39fc24:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T10:40:28,402 INFO [master/7ea43b39fc24:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-09T10:40:28,425 INFO [master/7ea43b39fc24:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 21ms. 2024-12-09T10:40:28,430 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36101-0x100bd55aca10001, quorum=127.0.0.1:50851, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T10:40:28,430 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35095-0x100bd55aca10000, quorum=127.0.0.1:50851, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T10:40:28,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741827_1003 (size=196) 2024-12-09T10:40:28,872 INFO [master/7ea43b39fc24:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T10:40:28,876 INFO [master/7ea43b39fc24:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-09T10:40:28,895 DEBUG [master/7ea43b39fc24:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:40:28,900 INFO [master/7ea43b39fc24:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T10:40:28,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741828_1004 (size=1189) 2024-12-09T10:40:29,362 INFO [master/7ea43b39fc24:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/MasterData/data/master/store 2024-12-09T10:40:29,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741829_1005 (size=34) 2024-12-09T10:40:29,390 INFO [master/7ea43b39fc24:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-09T10:40:29,394 DEBUG [master/7ea43b39fc24:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T10:40:29,395 DEBUG [master/7ea43b39fc24:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-09T10:40:29,395 INFO [master/7ea43b39fc24:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T10:40:29,395 DEBUG [master/7ea43b39fc24:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T10:40:29,397 DEBUG [master/7ea43b39fc24:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 1 ms 2024-12-09T10:40:29,397 DEBUG [master/7ea43b39fc24:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T10:40:29,397 INFO [master/7ea43b39fc24:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T10:40:29,399 DEBUG [master/7ea43b39fc24:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733740829395Disabling compacts and flushes for region at 1733740829395Disabling writes for close at 1733740829397 (+2 ms)Writing region close event to WAL at 1733740829397Closed at 1733740829397 2024-12-09T10:40:29,401 WARN [master/7ea43b39fc24:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/MasterData/data/master/store/.initializing 2024-12-09T10:40:29,401 DEBUG [master/7ea43b39fc24:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/MasterData/WALs/7ea43b39fc24,35095,1733740826918 2024-12-09T10:40:29,411 INFO [master/7ea43b39fc24:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T10:40:29,430 INFO [master/7ea43b39fc24:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=7ea43b39fc24%2C35095%2C1733740826918, suffix=, logDir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/MasterData/WALs/7ea43b39fc24,35095,1733740826918, archiveDir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/MasterData/oldWALs, maxLogs=10 2024-12-09T10:40:29,455 DEBUG [master/7ea43b39fc24:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/MasterData/WALs/7ea43b39fc24,35095,1733740826918/7ea43b39fc24%2C35095%2C1733740826918.1733740829436, exclude list is [], retry=0 2024-12-09T10:40:29,472 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34899,DS-334b36eb-fd29-4148-a54e-e73877f2c3da,DISK] 2024-12-09T10:40:29,475 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-09T10:40:29,510 INFO [master/7ea43b39fc24:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/MasterData/WALs/7ea43b39fc24,35095,1733740826918/7ea43b39fc24%2C35095%2C1733740826918.1733740829436 2024-12-09T10:40:29,511 DEBUG [master/7ea43b39fc24:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39659:39659)] 2024-12-09T10:40:29,512 DEBUG [master/7ea43b39fc24:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-09T10:40:29,512 DEBUG [master/7ea43b39fc24:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T10:40:29,515 DEBUG [master/7ea43b39fc24:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T10:40:29,516 DEBUG [master/7ea43b39fc24:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T10:40:29,554 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-09T10:40:29,581 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-09T10:40:29,585 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:40:29,588 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T10:40:29,589 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-09T10:40:29,593 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-09T10:40:29,593 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:40:29,595 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T10:40:29,595 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-09T10:40:29,598 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-09T10:40:29,598 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:40:29,599 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T10:40:29,599 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-09T10:40:29,602 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-09T10:40:29,602 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:40:29,603 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T10:40:29,603 DEBUG [master/7ea43b39fc24:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T10:40:29,608 DEBUG [master/7ea43b39fc24:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-09T10:40:29,610 DEBUG [master/7ea43b39fc24:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-09T10:40:29,616 DEBUG [master/7ea43b39fc24:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T10:40:29,616 DEBUG [master/7ea43b39fc24:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T10:40:29,620 DEBUG [master/7ea43b39fc24:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-09T10:40:29,624 DEBUG [master/7ea43b39fc24:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T10:40:29,629 DEBUG [master/7ea43b39fc24:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T10:40:29,631 INFO [master/7ea43b39fc24:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62489208, jitterRate=-0.06883823871612549}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-09T10:40:29,637 DEBUG [master/7ea43b39fc24:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733740829529Initializing all the Stores at 1733740829531 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733740829531Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733740829532 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733740829532Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733740829532Cleaning up temporary data from old regions at 1733740829616 (+84 ms)Region opened successfully at 1733740829637 (+21 ms) 2024-12-09T10:40:29,639 INFO [master/7ea43b39fc24:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-09T10:40:29,680 DEBUG [master/7ea43b39fc24:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4be0e44b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=7ea43b39fc24/172.17.0.3:0 2024-12-09T10:40:29,718 INFO [master/7ea43b39fc24:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-09T10:40:29,730 INFO [master/7ea43b39fc24:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-09T10:40:29,730 INFO [master/7ea43b39fc24:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-09T10:40:29,733 INFO [master/7ea43b39fc24:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-09T10:40:29,735 INFO [master/7ea43b39fc24:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-09T10:40:29,740 INFO [master/7ea43b39fc24:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-12-09T10:40:29,740 INFO [master/7ea43b39fc24:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-09T10:40:29,765 INFO [master/7ea43b39fc24:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-09T10:40:29,774 DEBUG [master/7ea43b39fc24:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35095-0x100bd55aca10000, quorum=127.0.0.1:50851, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-09T10:40:29,777 DEBUG [master/7ea43b39fc24:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-09T10:40:29,780 INFO [master/7ea43b39fc24:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-09T10:40:29,782 DEBUG [master/7ea43b39fc24:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35095-0x100bd55aca10000, quorum=127.0.0.1:50851, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-09T10:40:29,784 DEBUG [master/7ea43b39fc24:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-09T10:40:29,786 INFO [master/7ea43b39fc24:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-09T10:40:29,790 DEBUG [master/7ea43b39fc24:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35095-0x100bd55aca10000, quorum=127.0.0.1:50851, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-09T10:40:29,791 DEBUG [master/7ea43b39fc24:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-09T10:40:29,792 DEBUG [master/7ea43b39fc24:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35095-0x100bd55aca10000, quorum=127.0.0.1:50851, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-09T10:40:29,794 DEBUG [master/7ea43b39fc24:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-09T10:40:29,811 DEBUG [master/7ea43b39fc24:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35095-0x100bd55aca10000, quorum=127.0.0.1:50851, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-09T10:40:29,813 DEBUG [master/7ea43b39fc24:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-09T10:40:29,817 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35095-0x100bd55aca10000, quorum=127.0.0.1:50851, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-09T10:40:29,817 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36101-0x100bd55aca10001, quorum=127.0.0.1:50851, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-09T10:40:29,817 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35095-0x100bd55aca10000, quorum=127.0.0.1:50851, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T10:40:29,817 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36101-0x100bd55aca10001, quorum=127.0.0.1:50851, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T10:40:29,820 INFO [master/7ea43b39fc24:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=7ea43b39fc24,35095,1733740826918, sessionid=0x100bd55aca10000, setting cluster-up flag (Was=false) 2024-12-09T10:40:29,832 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35095-0x100bd55aca10000, quorum=127.0.0.1:50851, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T10:40:29,832 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36101-0x100bd55aca10001, quorum=127.0.0.1:50851, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T10:40:29,839 DEBUG [master/7ea43b39fc24:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-09T10:40:29,841 DEBUG [master/7ea43b39fc24:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=7ea43b39fc24,35095,1733740826918 2024-12-09T10:40:29,846 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35095-0x100bd55aca10000, quorum=127.0.0.1:50851, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T10:40:29,846 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36101-0x100bd55aca10001, quorum=127.0.0.1:50851, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T10:40:29,852 DEBUG [master/7ea43b39fc24:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-09T10:40:29,854 DEBUG [master/7ea43b39fc24:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=7ea43b39fc24,35095,1733740826918 2024-12-09T10:40:29,862 INFO [master/7ea43b39fc24:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-09T10:40:29,896 INFO [RS:0;7ea43b39fc24:36101 {}] regionserver.HRegionServer(746): ClusterId : 0945b5ae-b46b-4b8f-af48-56b0bb92df09 2024-12-09T10:40:29,898 DEBUG [RS:0;7ea43b39fc24:36101 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-09T10:40:29,903 DEBUG [RS:0;7ea43b39fc24:36101 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-09T10:40:29,903 DEBUG [RS:0;7ea43b39fc24:36101 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-09T10:40:29,906 DEBUG [RS:0;7ea43b39fc24:36101 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-09T10:40:29,907 DEBUG [RS:0;7ea43b39fc24:36101 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@542e6221, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=7ea43b39fc24/172.17.0.3:0 2024-12-09T10:40:29,922 DEBUG [RS:0;7ea43b39fc24:36101 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;7ea43b39fc24:36101 2024-12-09T10:40:29,925 INFO [RS:0;7ea43b39fc24:36101 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-09T10:40:29,926 INFO [RS:0;7ea43b39fc24:36101 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-09T10:40:29,926 DEBUG [RS:0;7ea43b39fc24:36101 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-09T10:40:29,928 INFO [RS:0;7ea43b39fc24:36101 {}] regionserver.HRegionServer(2659): reportForDuty to master=7ea43b39fc24,35095,1733740826918 with port=36101, startcode=1733740827678 2024-12-09T10:40:29,943 DEBUG [RS:0;7ea43b39fc24:36101 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-09T10:40:29,945 DEBUG [master/7ea43b39fc24:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-09T10:40:29,957 INFO [master/7ea43b39fc24:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-09T10:40:29,966 INFO [master/7ea43b39fc24:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-09T10:40:29,974 DEBUG [master/7ea43b39fc24:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 7ea43b39fc24,35095,1733740826918 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-09T10:40:29,980 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38507, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-09T10:40:29,987 DEBUG [master/7ea43b39fc24:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/7ea43b39fc24:0, corePoolSize=5, maxPoolSize=5 2024-12-09T10:40:29,988 DEBUG [master/7ea43b39fc24:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/7ea43b39fc24:0, corePoolSize=5, maxPoolSize=5 2024-12-09T10:40:29,988 DEBUG [master/7ea43b39fc24:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/7ea43b39fc24:0, corePoolSize=5, maxPoolSize=5 2024-12-09T10:40:29,988 DEBUG [master/7ea43b39fc24:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/7ea43b39fc24:0, corePoolSize=5, maxPoolSize=5 2024-12-09T10:40:29,988 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35095 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:29,989 DEBUG [master/7ea43b39fc24:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/7ea43b39fc24:0, corePoolSize=10, maxPoolSize=10 2024-12-09T10:40:29,989 DEBUG [master/7ea43b39fc24:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/7ea43b39fc24:0, corePoolSize=1, maxPoolSize=1 2024-12-09T10:40:29,989 DEBUG [master/7ea43b39fc24:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/7ea43b39fc24:0, corePoolSize=2, maxPoolSize=2 2024-12-09T10:40:29,989 DEBUG [master/7ea43b39fc24:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/7ea43b39fc24:0, corePoolSize=1, maxPoolSize=1 2024-12-09T10:40:29,993 INFO [master/7ea43b39fc24:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733740859993 2024-12-09T10:40:29,994 INFO [master/7ea43b39fc24:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-09T10:40:29,996 INFO [master/7ea43b39fc24:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-09T10:40:29,996 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-09T10:40:29,996 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-09T10:40:30,000 INFO [master/7ea43b39fc24:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-09T10:40:30,000 INFO [master/7ea43b39fc24:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-09T10:40:30,000 INFO [master/7ea43b39fc24:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-09T10:40:30,000 INFO [master/7ea43b39fc24:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-09T10:40:30,001 INFO [master/7ea43b39fc24:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-09T10:40:30,005 INFO [master/7ea43b39fc24:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-09T10:40:30,005 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:40:30,006 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-09T10:40:30,006 INFO [master/7ea43b39fc24:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-09T10:40:30,006 INFO [master/7ea43b39fc24:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-09T10:40:30,009 INFO [master/7ea43b39fc24:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-09T10:40:30,009 INFO [master/7ea43b39fc24:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-09T10:40:30,011 DEBUG [master/7ea43b39fc24:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/7ea43b39fc24:0:becomeActiveMaster-HFileCleaner.large.0-1733740830010,5,FailOnTimeoutGroup] 2024-12-09T10:40:30,013 DEBUG [master/7ea43b39fc24:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/7ea43b39fc24:0:becomeActiveMaster-HFileCleaner.small.0-1733740830011,5,FailOnTimeoutGroup] 2024-12-09T10:40:30,013 INFO [master/7ea43b39fc24:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-09T10:40:30,013 INFO [master/7ea43b39fc24:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-09T10:40:30,014 INFO [master/7ea43b39fc24:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-09T10:40:30,014 INFO [master/7ea43b39fc24:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-09T10:40:30,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741831_1007 (size=1321) 2024-12-09T10:40:30,022 DEBUG [RS:0;7ea43b39fc24:36101 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-12-09T10:40:30,022 WARN [RS:0;7ea43b39fc24:36101 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-12-09T10:40:30,124 INFO [RS:0;7ea43b39fc24:36101 {}] regionserver.HRegionServer(2659): reportForDuty to master=7ea43b39fc24,35095,1733740826918 with port=36101, startcode=1733740827678 2024-12-09T10:40:30,126 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35095 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:30,129 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35095 {}] master.ServerManager(517): Registering regionserver=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:30,138 DEBUG [RS:0;7ea43b39fc24:36101 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160 2024-12-09T10:40:30,138 DEBUG [RS:0;7ea43b39fc24:36101 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:34553 2024-12-09T10:40:30,138 DEBUG [RS:0;7ea43b39fc24:36101 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-09T10:40:30,142 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35095-0x100bd55aca10000, quorum=127.0.0.1:50851, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-09T10:40:30,143 DEBUG [RS:0;7ea43b39fc24:36101 {}] zookeeper.ZKUtil(111): regionserver:36101-0x100bd55aca10001, quorum=127.0.0.1:50851, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:30,143 WARN [RS:0;7ea43b39fc24:36101 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-09T10:40:30,143 INFO [RS:0;7ea43b39fc24:36101 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T10:40:30,143 DEBUG [RS:0;7ea43b39fc24:36101 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/WALs/7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:30,145 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [7ea43b39fc24,36101,1733740827678] 2024-12-09T10:40:30,170 INFO [RS:0;7ea43b39fc24:36101 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-09T10:40:30,184 INFO [RS:0;7ea43b39fc24:36101 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-09T10:40:30,189 INFO [RS:0;7ea43b39fc24:36101 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-09T10:40:30,190 INFO [RS:0;7ea43b39fc24:36101 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T10:40:30,191 INFO [RS:0;7ea43b39fc24:36101 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-09T10:40:30,196 INFO [RS:0;7ea43b39fc24:36101 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-09T10:40:30,198 INFO [RS:0;7ea43b39fc24:36101 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-09T10:40:30,198 DEBUG [RS:0;7ea43b39fc24:36101 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/7ea43b39fc24:0, corePoolSize=1, maxPoolSize=1 2024-12-09T10:40:30,198 DEBUG [RS:0;7ea43b39fc24:36101 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/7ea43b39fc24:0, corePoolSize=1, maxPoolSize=1 2024-12-09T10:40:30,198 DEBUG [RS:0;7ea43b39fc24:36101 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/7ea43b39fc24:0, corePoolSize=1, maxPoolSize=1 2024-12-09T10:40:30,199 DEBUG [RS:0;7ea43b39fc24:36101 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/7ea43b39fc24:0, corePoolSize=1, maxPoolSize=1 2024-12-09T10:40:30,199 DEBUG [RS:0;7ea43b39fc24:36101 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/7ea43b39fc24:0, corePoolSize=1, maxPoolSize=1 2024-12-09T10:40:30,199 DEBUG [RS:0;7ea43b39fc24:36101 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/7ea43b39fc24:0, corePoolSize=2, maxPoolSize=2 2024-12-09T10:40:30,199 DEBUG [RS:0;7ea43b39fc24:36101 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/7ea43b39fc24:0, corePoolSize=1, maxPoolSize=1 2024-12-09T10:40:30,199 DEBUG [RS:0;7ea43b39fc24:36101 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/7ea43b39fc24:0, corePoolSize=1, maxPoolSize=1 2024-12-09T10:40:30,199 DEBUG [RS:0;7ea43b39fc24:36101 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/7ea43b39fc24:0, corePoolSize=1, maxPoolSize=1 2024-12-09T10:40:30,200 DEBUG [RS:0;7ea43b39fc24:36101 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/7ea43b39fc24:0, corePoolSize=1, maxPoolSize=1 2024-12-09T10:40:30,200 DEBUG [RS:0;7ea43b39fc24:36101 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/7ea43b39fc24:0, corePoolSize=1, maxPoolSize=1 2024-12-09T10:40:30,200 DEBUG [RS:0;7ea43b39fc24:36101 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/7ea43b39fc24:0, corePoolSize=1, maxPoolSize=1 2024-12-09T10:40:30,200 DEBUG [RS:0;7ea43b39fc24:36101 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/7ea43b39fc24:0, corePoolSize=3, maxPoolSize=3 2024-12-09T10:40:30,200 DEBUG [RS:0;7ea43b39fc24:36101 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0, corePoolSize=3, maxPoolSize=3 2024-12-09T10:40:30,202 INFO [RS:0;7ea43b39fc24:36101 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-09T10:40:30,202 INFO [RS:0;7ea43b39fc24:36101 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-09T10:40:30,202 INFO [RS:0;7ea43b39fc24:36101 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T10:40:30,203 INFO [RS:0;7ea43b39fc24:36101 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-09T10:40:30,203 INFO [RS:0;7ea43b39fc24:36101 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-09T10:40:30,203 INFO [RS:0;7ea43b39fc24:36101 {}] hbase.ChoreService(168): Chore ScheduledChore name=7ea43b39fc24,36101,1733740827678-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-09T10:40:30,228 INFO [RS:0;7ea43b39fc24:36101 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-09T10:40:30,230 INFO [RS:0;7ea43b39fc24:36101 {}] hbase.ChoreService(168): Chore ScheduledChore name=7ea43b39fc24,36101,1733740827678-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T10:40:30,230 INFO [RS:0;7ea43b39fc24:36101 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T10:40:30,231 INFO [RS:0;7ea43b39fc24:36101 {}] regionserver.Replication(171): 7ea43b39fc24,36101,1733740827678 started 2024-12-09T10:40:30,253 INFO [RS:0;7ea43b39fc24:36101 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T10:40:30,253 INFO [RS:0;7ea43b39fc24:36101 {}] regionserver.HRegionServer(1482): Serving as 7ea43b39fc24,36101,1733740827678, RpcServer on 7ea43b39fc24/172.17.0.3:36101, sessionid=0x100bd55aca10001 2024-12-09T10:40:30,254 DEBUG [RS:0;7ea43b39fc24:36101 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-09T10:40:30,255 DEBUG [RS:0;7ea43b39fc24:36101 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:30,255 DEBUG [RS:0;7ea43b39fc24:36101 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '7ea43b39fc24,36101,1733740827678' 2024-12-09T10:40:30,255 DEBUG [RS:0;7ea43b39fc24:36101 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-09T10:40:30,256 DEBUG [RS:0;7ea43b39fc24:36101 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-09T10:40:30,257 DEBUG [RS:0;7ea43b39fc24:36101 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-09T10:40:30,257 DEBUG [RS:0;7ea43b39fc24:36101 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-09T10:40:30,258 DEBUG [RS:0;7ea43b39fc24:36101 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:30,258 DEBUG [RS:0;7ea43b39fc24:36101 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '7ea43b39fc24,36101,1733740827678' 2024-12-09T10:40:30,258 DEBUG [RS:0;7ea43b39fc24:36101 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-09T10:40:30,259 DEBUG [RS:0;7ea43b39fc24:36101 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-09T10:40:30,259 DEBUG [RS:0;7ea43b39fc24:36101 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-09T10:40:30,259 INFO [RS:0;7ea43b39fc24:36101 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-09T10:40:30,259 INFO [RS:0;7ea43b39fc24:36101 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-09T10:40:30,365 INFO [RS:0;7ea43b39fc24:36101 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T10:40:30,369 INFO [RS:0;7ea43b39fc24:36101 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=7ea43b39fc24%2C36101%2C1733740827678, suffix=, logDir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/WALs/7ea43b39fc24,36101,1733740827678, archiveDir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/oldWALs, maxLogs=32 2024-12-09T10:40:30,386 DEBUG [RS:0;7ea43b39fc24:36101 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/WALs/7ea43b39fc24,36101,1733740827678/7ea43b39fc24%2C36101%2C1733740827678.1733740830371, exclude list is [], retry=0 2024-12-09T10:40:30,391 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34899,DS-334b36eb-fd29-4148-a54e-e73877f2c3da,DISK] 2024-12-09T10:40:30,394 INFO [RS:0;7ea43b39fc24:36101 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/WALs/7ea43b39fc24,36101,1733740827678/7ea43b39fc24%2C36101%2C1733740827678.1733740830371 2024-12-09T10:40:30,395 DEBUG [RS:0;7ea43b39fc24:36101 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39659:39659)] 2024-12-09T10:40:30,419 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-09T10:40:30,419 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160 2024-12-09T10:40:30,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741833_1009 (size=32) 2024-12-09T10:40:30,831 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T10:40:30,833 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-09T10:40:30,836 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-09T10:40:30,836 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:40:30,837 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T10:40:30,837 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-09T10:40:30,840 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-09T10:40:30,840 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:40:30,841 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T10:40:30,841 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-09T10:40:30,843 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-09T10:40:30,843 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:40:30,844 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T10:40:30,845 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-09T10:40:30,847 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-09T10:40:30,847 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:40:30,848 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T10:40:30,848 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-09T10:40:30,849 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/hbase/meta/1588230740 2024-12-09T10:40:30,850 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/hbase/meta/1588230740 2024-12-09T10:40:30,852 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-09T10:40:30,852 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-09T10:40:30,853 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-09T10:40:30,855 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-09T10:40:30,858 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T10:40:30,859 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59083617, jitterRate=-0.11958549916744232}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-09T10:40:30,861 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733740830831Initializing all the Stores at 1733740830833 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733740830833Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733740830833Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733740830833Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733740830833Cleaning up temporary data from old regions at 1733740830852 (+19 ms)Region opened successfully at 1733740830861 (+9 ms) 2024-12-09T10:40:30,862 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-09T10:40:30,862 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-09T10:40:30,862 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-09T10:40:30,862 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-09T10:40:30,862 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-09T10:40:30,863 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-09T10:40:30,863 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733740830861Disabling compacts and flushes for region at 1733740830861Disabling writes for close at 1733740830862 (+1 ms)Writing region close event to WAL at 1733740830863 (+1 ms)Closed at 1733740830863 2024-12-09T10:40:30,867 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-09T10:40:30,867 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-09T10:40:30,873 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-09T10:40:30,882 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-09T10:40:30,884 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-09T10:40:31,037 DEBUG [7ea43b39fc24:35095 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-09T10:40:31,049 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:31,056 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7ea43b39fc24,36101,1733740827678, state=OPENING 2024-12-09T10:40:31,061 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-09T10:40:31,063 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36101-0x100bd55aca10001, quorum=127.0.0.1:50851, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T10:40:31,063 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35095-0x100bd55aca10000, quorum=127.0.0.1:50851, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T10:40:31,064 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T10:40:31,064 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T10:40:31,066 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-09T10:40:31,068 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=7ea43b39fc24,36101,1733740827678}] 2024-12-09T10:40:31,244 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-09T10:40:31,247 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51503, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-09T10:40:31,259 INFO [RS_OPEN_META-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-09T10:40:31,260 INFO [RS_OPEN_META-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T10:40:31,261 INFO [RS_OPEN_META-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-09T10:40:31,264 INFO [RS_OPEN_META-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=7ea43b39fc24%2C36101%2C1733740827678.meta, suffix=.meta, logDir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/WALs/7ea43b39fc24,36101,1733740827678, archiveDir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/oldWALs, maxLogs=32 2024-12-09T10:40:31,280 DEBUG [RS_OPEN_META-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/WALs/7ea43b39fc24,36101,1733740827678/7ea43b39fc24%2C36101%2C1733740827678.meta.1733740831265.meta, exclude list is [], retry=0 2024-12-09T10:40:31,284 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34899,DS-334b36eb-fd29-4148-a54e-e73877f2c3da,DISK] 2024-12-09T10:40:31,287 INFO [RS_OPEN_META-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/WALs/7ea43b39fc24,36101,1733740827678/7ea43b39fc24%2C36101%2C1733740827678.meta.1733740831265.meta 2024-12-09T10:40:31,287 DEBUG [RS_OPEN_META-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39659:39659)] 2024-12-09T10:40:31,288 DEBUG [RS_OPEN_META-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-09T10:40:31,290 DEBUG [RS_OPEN_META-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-09T10:40:31,292 DEBUG [RS_OPEN_META-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-09T10:40:31,297 INFO [RS_OPEN_META-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-09T10:40:31,301 DEBUG [RS_OPEN_META-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-09T10:40:31,301 DEBUG [RS_OPEN_META-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T10:40:31,302 DEBUG [RS_OPEN_META-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-09T10:40:31,302 DEBUG [RS_OPEN_META-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-09T10:40:31,305 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-09T10:40:31,307 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-09T10:40:31,307 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:40:31,308 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T10:40:31,308 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-09T10:40:31,310 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-09T10:40:31,310 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:40:31,311 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T10:40:31,311 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-09T10:40:31,312 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-09T10:40:31,312 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:40:31,313 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T10:40:31,313 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-09T10:40:31,315 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-09T10:40:31,315 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:40:31,316 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T10:40:31,316 DEBUG [RS_OPEN_META-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-09T10:40:31,318 DEBUG [RS_OPEN_META-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/hbase/meta/1588230740 2024-12-09T10:40:31,321 DEBUG [RS_OPEN_META-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/hbase/meta/1588230740 2024-12-09T10:40:31,324 DEBUG [RS_OPEN_META-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-09T10:40:31,324 DEBUG [RS_OPEN_META-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-09T10:40:31,325 DEBUG [RS_OPEN_META-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-09T10:40:31,327 DEBUG [RS_OPEN_META-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-09T10:40:31,329 INFO [RS_OPEN_META-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63355460, jitterRate=-0.05593007802963257}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-09T10:40:31,329 DEBUG [RS_OPEN_META-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-09T10:40:31,330 DEBUG [RS_OPEN_META-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733740831302Writing region info on filesystem at 1733740831302Initializing all the Stores at 1733740831304 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733740831304Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733740831305 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733740831305Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733740831305Cleaning up temporary data from old regions at 1733740831324 (+19 ms)Running coprocessor post-open hooks at 1733740831329 (+5 ms)Region opened successfully at 1733740831330 (+1 ms) 2024-12-09T10:40:31,338 INFO [RS_OPEN_META-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733740831234 2024-12-09T10:40:31,349 DEBUG [RS_OPEN_META-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-09T10:40:31,350 INFO [RS_OPEN_META-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-09T10:40:31,352 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:31,354 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7ea43b39fc24,36101,1733740827678, state=OPEN 2024-12-09T10:40:31,360 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36101-0x100bd55aca10001, quorum=127.0.0.1:50851, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T10:40:31,360 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35095-0x100bd55aca10000, quorum=127.0.0.1:50851, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T10:40:31,361 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T10:40:31,361 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T10:40:31,361 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:31,367 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-09T10:40:31,367 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=7ea43b39fc24,36101,1733740827678 in 294 msec 2024-12-09T10:40:31,374 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-09T10:40:31,375 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 496 msec 2024-12-09T10:40:31,376 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-09T10:40:31,376 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-09T10:40:31,397 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T10:40:31,399 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ea43b39fc24,36101,1733740827678, seqNum=-1] 2024-12-09T10:40:31,420 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T10:40:31,423 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59155, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T10:40:31,444 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.5490 sec 2024-12-09T10:40:31,444 INFO [master/7ea43b39fc24:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733740831444, completionTime=-1 2024-12-09T10:40:31,447 INFO [master/7ea43b39fc24:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-12-09T10:40:31,447 DEBUG [master/7ea43b39fc24:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-09T10:40:31,473 INFO [master/7ea43b39fc24:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-12-09T10:40:31,474 INFO [master/7ea43b39fc24:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733740891473 2024-12-09T10:40:31,474 INFO [master/7ea43b39fc24:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733740951474 2024-12-09T10:40:31,474 INFO [master/7ea43b39fc24:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 26 msec 2024-12-09T10:40:31,476 INFO [master/7ea43b39fc24:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7ea43b39fc24,35095,1733740826918-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T10:40:31,476 INFO [master/7ea43b39fc24:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7ea43b39fc24,35095,1733740826918-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T10:40:31,476 INFO [master/7ea43b39fc24:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7ea43b39fc24,35095,1733740826918-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T10:40:31,478 INFO [master/7ea43b39fc24:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-7ea43b39fc24:35095, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T10:40:31,478 INFO [master/7ea43b39fc24:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-09T10:40:31,479 INFO [master/7ea43b39fc24:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-09T10:40:31,484 DEBUG [master/7ea43b39fc24:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-09T10:40:31,505 INFO [master/7ea43b39fc24:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.649sec 2024-12-09T10:40:31,507 INFO [master/7ea43b39fc24:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-09T10:40:31,508 INFO [master/7ea43b39fc24:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-09T10:40:31,509 INFO [master/7ea43b39fc24:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-09T10:40:31,509 INFO [master/7ea43b39fc24:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-09T10:40:31,510 INFO [master/7ea43b39fc24:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-09T10:40:31,510 INFO [master/7ea43b39fc24:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7ea43b39fc24,35095,1733740826918-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-09T10:40:31,511 INFO [master/7ea43b39fc24:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7ea43b39fc24,35095,1733740826918-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-09T10:40:31,519 DEBUG [master/7ea43b39fc24:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-09T10:40:31,520 INFO [master/7ea43b39fc24:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-09T10:40:31,521 INFO [master/7ea43b39fc24:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7ea43b39fc24,35095,1733740826918-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T10:40:31,606 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e632805, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:40:31,608 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-09T10:40:31,608 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-09T10:40:31,611 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ea43b39fc24,35095,-1 for getting cluster id 2024-12-09T10:40:31,614 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T10:40:31,622 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0945b5ae-b46b-4b8f-af48-56b0bb92df09' 2024-12-09T10:40:31,624 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T10:40:31,624 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0945b5ae-b46b-4b8f-af48-56b0bb92df09" 2024-12-09T10:40:31,627 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5db9d8a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:40:31,627 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ea43b39fc24,35095,-1] 2024-12-09T10:40:31,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T10:40:31,631 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:40:31,633 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39522, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T10:40:31,635 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f999ad2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:40:31,636 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T10:40:31,643 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ea43b39fc24,36101,1733740827678, seqNum=-1] 2024-12-09T10:40:31,644 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T10:40:31,647 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46308, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T10:40:31,685 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=7ea43b39fc24,35095,1733740826918 2024-12-09T10:40:31,702 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=330, ProcessCount=11, AvailableMemoryMB=8650 2024-12-09T10:40:31,715 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-09T10:40:31,721 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 7ea43b39fc24,35095,1733740826918 2024-12-09T10:40:31,724 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5b65baa9 2024-12-09T10:40:31,726 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-09T10:40:31,729 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39534, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-09T10:40:31,731 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-09T10:40:31,735 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T10:40:31,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-09T10:40:31,745 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-09T10:40:31,747 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-12-09T10:40:31,747 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:40:31,750 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-09T10:40:31,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-09T10:40:31,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741835_1011 (size=1061) 2024-12-09T10:40:31,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-09T10:40:32,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-09T10:40:32,166 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160 2024-12-09T10:40:32,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741836_1012 (size=53) 2024-12-09T10:40:32,175 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T10:40:32,175 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 1d9d6bba7dcd09663c7b08474fc1c1b5, disabling compactions & flushes 2024-12-09T10:40:32,175 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:32,175 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:32,175 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. after waiting 0 ms 2024-12-09T10:40:32,175 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:32,175 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:32,175 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 1d9d6bba7dcd09663c7b08474fc1c1b5: Waiting for close lock at 1733740832175Disabling compacts and flushes for region at 1733740832175Disabling writes for close at 1733740832175Writing region close event to WAL at 1733740832175Closed at 1733740832175 2024-12-09T10:40:32,178 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-09T10:40:32,183 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733740832178"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733740832178"}]},"ts":"1733740832178"} 2024-12-09T10:40:32,189 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-09T10:40:32,191 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-09T10:40:32,194 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733740832191"}]},"ts":"1733740832191"} 2024-12-09T10:40:32,199 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-09T10:40:32,201 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1d9d6bba7dcd09663c7b08474fc1c1b5, ASSIGN}] 2024-12-09T10:40:32,204 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1d9d6bba7dcd09663c7b08474fc1c1b5, ASSIGN 2024-12-09T10:40:32,206 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1d9d6bba7dcd09663c7b08474fc1c1b5, ASSIGN; state=OFFLINE, location=7ea43b39fc24,36101,1733740827678; forceNewPlan=false, retain=false 2024-12-09T10:40:32,358 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=1d9d6bba7dcd09663c7b08474fc1c1b5, regionState=OPENING, regionLocation=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:32,363 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1d9d6bba7dcd09663c7b08474fc1c1b5, ASSIGN because future has completed 2024-12-09T10:40:32,364 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678}] 2024-12-09T10:40:32,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-09T10:40:32,525 INFO [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:32,525 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} 2024-12-09T10:40:32,526 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 1d9d6bba7dcd09663c7b08474fc1c1b5 2024-12-09T10:40:32,526 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T10:40:32,526 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 1d9d6bba7dcd09663c7b08474fc1c1b5 2024-12-09T10:40:32,526 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 1d9d6bba7dcd09663c7b08474fc1c1b5 2024-12-09T10:40:32,528 INFO [StoreOpener-1d9d6bba7dcd09663c7b08474fc1c1b5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 1d9d6bba7dcd09663c7b08474fc1c1b5 2024-12-09T10:40:32,531 INFO [StoreOpener-1d9d6bba7dcd09663c7b08474fc1c1b5-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T10:40:32,532 INFO [StoreOpener-1d9d6bba7dcd09663c7b08474fc1c1b5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1d9d6bba7dcd09663c7b08474fc1c1b5 columnFamilyName A 2024-12-09T10:40:32,532 DEBUG [StoreOpener-1d9d6bba7dcd09663c7b08474fc1c1b5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:40:32,533 INFO [StoreOpener-1d9d6bba7dcd09663c7b08474fc1c1b5-1 {}] regionserver.HStore(327): Store=1d9d6bba7dcd09663c7b08474fc1c1b5/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T10:40:32,533 INFO [StoreOpener-1d9d6bba7dcd09663c7b08474fc1c1b5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 1d9d6bba7dcd09663c7b08474fc1c1b5 2024-12-09T10:40:32,535 INFO [StoreOpener-1d9d6bba7dcd09663c7b08474fc1c1b5-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T10:40:32,535 INFO [StoreOpener-1d9d6bba7dcd09663c7b08474fc1c1b5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1d9d6bba7dcd09663c7b08474fc1c1b5 columnFamilyName B 2024-12-09T10:40:32,535 DEBUG [StoreOpener-1d9d6bba7dcd09663c7b08474fc1c1b5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:40:32,536 INFO [StoreOpener-1d9d6bba7dcd09663c7b08474fc1c1b5-1 {}] regionserver.HStore(327): Store=1d9d6bba7dcd09663c7b08474fc1c1b5/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T10:40:32,536 INFO [StoreOpener-1d9d6bba7dcd09663c7b08474fc1c1b5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 1d9d6bba7dcd09663c7b08474fc1c1b5 2024-12-09T10:40:32,538 INFO [StoreOpener-1d9d6bba7dcd09663c7b08474fc1c1b5-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T10:40:32,539 INFO [StoreOpener-1d9d6bba7dcd09663c7b08474fc1c1b5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1d9d6bba7dcd09663c7b08474fc1c1b5 columnFamilyName C 2024-12-09T10:40:32,539 DEBUG [StoreOpener-1d9d6bba7dcd09663c7b08474fc1c1b5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:40:32,539 INFO [StoreOpener-1d9d6bba7dcd09663c7b08474fc1c1b5-1 {}] regionserver.HStore(327): Store=1d9d6bba7dcd09663c7b08474fc1c1b5/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T10:40:32,540 INFO [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:32,540 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 1d9d6bba7dcd09663c7b08474fc1c1b5 2024-12-09T10:40:32,542 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5 2024-12-09T10:40:32,543 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5 2024-12-09T10:40:32,545 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 1d9d6bba7dcd09663c7b08474fc1c1b5 2024-12-09T10:40:32,545 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 1d9d6bba7dcd09663c7b08474fc1c1b5 2024-12-09T10:40:32,546 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-09T10:40:32,549 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 1d9d6bba7dcd09663c7b08474fc1c1b5 2024-12-09T10:40:32,552 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T10:40:32,553 INFO [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 1d9d6bba7dcd09663c7b08474fc1c1b5; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67607712, jitterRate=0.007433414459228516}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-09T10:40:32,553 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1d9d6bba7dcd09663c7b08474fc1c1b5 2024-12-09T10:40:32,554 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 1d9d6bba7dcd09663c7b08474fc1c1b5: Running coprocessor pre-open hook at 1733740832526Writing region info on filesystem at 1733740832526Initializing all the Stores at 1733740832528 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733740832528Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733740832528Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733740832528Cleaning up temporary data from old regions at 1733740832545 (+17 ms)Running coprocessor post-open hooks at 1733740832553 (+8 ms)Region opened successfully at 1733740832554 (+1 ms) 2024-12-09T10:40:32,557 INFO [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., pid=6, masterSystemTime=1733740832518 2024-12-09T10:40:32,561 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:32,561 INFO [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:32,562 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=1d9d6bba7dcd09663c7b08474fc1c1b5, regionState=OPEN, openSeqNum=2, regionLocation=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:32,567 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 because future has completed 2024-12-09T10:40:32,573 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-09T10:40:32,573 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 in 205 msec 2024-12-09T10:40:32,577 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-09T10:40:32,577 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1d9d6bba7dcd09663c7b08474fc1c1b5, ASSIGN in 372 msec 2024-12-09T10:40:32,579 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-09T10:40:32,579 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733740832579"}]},"ts":"1733740832579"} 2024-12-09T10:40:32,583 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-09T10:40:32,585 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-09T10:40:32,588 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 848 msec 2024-12-09T10:40:32,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-09T10:40:32,891 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-09T10:40:32,897 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14b5ae32, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:40:32,897 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ea43b39fc24,35095,-1 for getting cluster id 2024-12-09T10:40:32,897 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T10:40:32,899 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0945b5ae-b46b-4b8f-af48-56b0bb92df09' 2024-12-09T10:40:32,899 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T10:40:32,899 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0945b5ae-b46b-4b8f-af48-56b0bb92df09" 2024-12-09T10:40:32,900 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@411c73ea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:40:32,900 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ea43b39fc24,35095,-1] 2024-12-09T10:40:32,900 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T10:40:32,900 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:40:32,902 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39554, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T10:40:32,904 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6825ee03, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:40:32,904 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T10:40:32,906 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ea43b39fc24,36101,1733740827678, seqNum=-1] 2024-12-09T10:40:32,906 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T10:40:32,909 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46324, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T10:40:32,912 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-09T10:40:32,914 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 7ea43b39fc24,35095,1733740826918 2024-12-09T10:40:32,914 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1b005fc9 2024-12-09T10:40:32,917 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-09T10:40:32,919 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39570, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-09T10:40:32,925 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59d01ad2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:40:32,925 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ea43b39fc24,35095,-1 for getting cluster id 2024-12-09T10:40:32,926 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T10:40:32,927 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0945b5ae-b46b-4b8f-af48-56b0bb92df09' 2024-12-09T10:40:32,928 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T10:40:32,928 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0945b5ae-b46b-4b8f-af48-56b0bb92df09" 2024-12-09T10:40:32,928 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b130fb1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:40:32,928 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ea43b39fc24,35095,-1] 2024-12-09T10:40:32,929 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T10:40:32,929 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:40:32,931 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39576, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T10:40:32,932 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d1d7645, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:40:32,933 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6fbe41fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:40:32,933 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ea43b39fc24,35095,-1 for getting cluster id 2024-12-09T10:40:32,934 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T10:40:32,935 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0945b5ae-b46b-4b8f-af48-56b0bb92df09' 2024-12-09T10:40:32,935 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T10:40:32,936 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0945b5ae-b46b-4b8f-af48-56b0bb92df09" 2024-12-09T10:40:32,936 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27ff6f12, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:40:32,936 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ea43b39fc24,35095,-1] 2024-12-09T10:40:32,936 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T10:40:32,937 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:40:32,938 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39596, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T10:40:32,940 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10bc6ecb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:40:32,940 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57c2ff46, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:40:32,941 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ea43b39fc24,35095,-1 for getting cluster id 2024-12-09T10:40:32,941 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T10:40:32,942 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0945b5ae-b46b-4b8f-af48-56b0bb92df09' 2024-12-09T10:40:32,942 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T10:40:32,943 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0945b5ae-b46b-4b8f-af48-56b0bb92df09" 2024-12-09T10:40:32,943 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ff1f1f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:40:32,943 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ea43b39fc24,35095,-1] 2024-12-09T10:40:32,943 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T10:40:32,944 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:40:32,945 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39620, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T10:40:32,946 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b37f010, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:40:32,947 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@493fccb8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:40:32,947 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ea43b39fc24,35095,-1 for getting cluster id 2024-12-09T10:40:32,947 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T10:40:32,949 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0945b5ae-b46b-4b8f-af48-56b0bb92df09' 2024-12-09T10:40:32,949 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T10:40:32,949 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0945b5ae-b46b-4b8f-af48-56b0bb92df09" 2024-12-09T10:40:32,950 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38788dd6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:40:32,950 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ea43b39fc24,35095,-1] 2024-12-09T10:40:32,950 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T10:40:32,951 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:40:32,953 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39638, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T10:40:32,954 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35818969, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:40:32,955 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73985d6f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:40:32,955 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ea43b39fc24,35095,-1 for getting cluster id 2024-12-09T10:40:32,955 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T10:40:32,956 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0945b5ae-b46b-4b8f-af48-56b0bb92df09' 2024-12-09T10:40:32,956 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T10:40:32,956 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0945b5ae-b46b-4b8f-af48-56b0bb92df09" 2024-12-09T10:40:32,957 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9a6bf59, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:40:32,957 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ea43b39fc24,35095,-1] 2024-12-09T10:40:32,957 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T10:40:32,958 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:40:32,959 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39660, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T10:40:32,960 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54c50a81, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:40:32,961 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@224eddc9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:40:32,961 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ea43b39fc24,35095,-1 for getting cluster id 2024-12-09T10:40:32,961 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T10:40:32,962 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0945b5ae-b46b-4b8f-af48-56b0bb92df09' 2024-12-09T10:40:32,963 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T10:40:32,963 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0945b5ae-b46b-4b8f-af48-56b0bb92df09" 2024-12-09T10:40:32,964 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7af5f6a7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:40:32,964 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ea43b39fc24,35095,-1] 2024-12-09T10:40:32,964 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T10:40:32,964 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:40:32,965 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39676, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T10:40:32,966 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32343cb0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:40:32,967 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3eab186f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:40:32,967 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ea43b39fc24,35095,-1 for getting cluster id 2024-12-09T10:40:32,967 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T10:40:32,968 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0945b5ae-b46b-4b8f-af48-56b0bb92df09' 2024-12-09T10:40:32,968 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T10:40:32,968 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0945b5ae-b46b-4b8f-af48-56b0bb92df09" 2024-12-09T10:40:32,969 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@af61fa8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:40:32,969 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ea43b39fc24,35095,-1] 2024-12-09T10:40:32,969 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T10:40:32,969 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:40:32,970 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39694, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T10:40:32,971 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6db01178, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:40:32,972 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61de84df, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:40:32,972 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ea43b39fc24,35095,-1 for getting cluster id 2024-12-09T10:40:32,972 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T10:40:32,973 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0945b5ae-b46b-4b8f-af48-56b0bb92df09' 2024-12-09T10:40:32,973 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T10:40:32,973 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0945b5ae-b46b-4b8f-af48-56b0bb92df09" 2024-12-09T10:40:32,974 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f7cbb29, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:40:32,974 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ea43b39fc24,35095,-1] 2024-12-09T10:40:32,974 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T10:40:32,974 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:40:32,975 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39722, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T10:40:32,976 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76c8af47, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:40:32,977 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4bafb7e3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:40:32,977 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ea43b39fc24,35095,-1 for getting cluster id 2024-12-09T10:40:32,977 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T10:40:32,979 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0945b5ae-b46b-4b8f-af48-56b0bb92df09' 2024-12-09T10:40:32,979 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T10:40:32,980 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0945b5ae-b46b-4b8f-af48-56b0bb92df09" 2024-12-09T10:40:32,980 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13c29b6b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:40:32,980 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ea43b39fc24,35095,-1] 2024-12-09T10:40:32,980 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T10:40:32,980 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:40:32,981 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39750, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T10:40:32,982 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78aaa9c8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:40:32,986 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T10:40:32,986 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T10:40:32,986 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T10:40:32,986 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T10:40:32,986 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T10:40:32,986 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T10:40:32,989 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ea43b39fc24,36101,1733740827678, seqNum=-1] 2024-12-09T10:40:32,989 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ea43b39fc24,36101,1733740827678, seqNum=-1] 2024-12-09T10:40:32,989 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ea43b39fc24,36101,1733740827678, seqNum=-1] 2024-12-09T10:40:32,989 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ea43b39fc24,36101,1733740827678, seqNum=-1] 2024-12-09T10:40:32,989 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T10:40:32,989 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T10:40:32,989 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T10:40:32,990 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T10:40:32,990 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ea43b39fc24,36101,1733740827678, seqNum=-1] 2024-12-09T10:40:32,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T10:40:32,991 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46336, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T10:40:32,992 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46344, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T10:40:32,992 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-09T10:40:32,992 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46346, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T10:40:32,992 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46362, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T10:40:32,992 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46374, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T10:40:32,993 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T10:40:32,993 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ea43b39fc24,36101,1733740827678, seqNum=-1] 2024-12-09T10:40:32,993 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T10:40:32,995 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T10:40:32,995 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ea43b39fc24,36101,1733740827678, seqNum=-1] 2024-12-09T10:40:32,995 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46390, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T10:40:32,995 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T10:40:32,998 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46396, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T10:40:33,001 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2] 2024-12-09T10:40:33,002 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2] 2024-12-09T10:40:33,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-12-09T10:40:33,005 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ea43b39fc24,36101,1733740827678, seqNum=-1] 2024-12-09T10:40:33,005 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T10:40:33,006 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T10:40:33,007 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2] 2024-12-09T10:40:33,007 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2] 2024-12-09T10:40:33,008 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46398, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T10:40:33,009 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2] 2024-12-09T10:40:33,010 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ea43b39fc24,36101,1733740827678, seqNum=-1] 2024-12-09T10:40:33,011 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T10:40:33,013 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T10:40:33,013 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46412, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T10:40:33,014 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2] 2024-12-09T10:40:33,015 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2] 2024-12-09T10:40:33,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2] 2024-12-09T10:40:33,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-09T10:40:33,021 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T10:40:33,022 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2] 2024-12-09T10:40:33,023 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T10:40:33,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(8855): Flush requested on 1d9d6bba7dcd09663c7b08474fc1c1b5 2024-12-09T10:40:33,084 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1d9d6bba7dcd09663c7b08474fc1c1b5 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-09T10:40:33,100 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d9d6bba7dcd09663c7b08474fc1c1b5, store=A 2024-12-09T10:40:33,100 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:40:33,100 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d9d6bba7dcd09663c7b08474fc1c1b5, store=B 2024-12-09T10:40:33,100 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:40:33,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d9d6bba7dcd09663c7b08474fc1c1b5, store=C 2024-12-09T10:40:33,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:40:33,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-09T10:40:33,204 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-09T10:40:33,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:33,210 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/A/0cf007a7125d4686b4d48808b19fa98b is 50, key is test_row_0/A:col10/1733740833062/Put/seqid=0 2024-12-09T10:40:33,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. as already flushing 2024-12-09T10:40:33,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:33,211 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:40:33,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:40:33,223 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:33,224 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:33,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46396 deadline: 1733740893202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:33,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46390 deadline: 1733740893198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:33,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:33,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:33,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:33,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:40:33,242 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:33,243 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:33,243 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:33,244 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:33,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46398 deadline: 1733740893221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:33,246 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:33,246 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:33,247 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:33,247 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:33,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46362 deadline: 1733740893225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:33,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741837_1013 (size=12001) 2024-12-09T10:40:33,248 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:33,248 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:33,248 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:33,248 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:33,248 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/A/0cf007a7125d4686b4d48808b19fa98b 2024-12-09T10:40:33,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46374 deadline: 1733740893227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:33,250 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:33,251 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:33,251 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:33,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-09T10:40:33,349 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:33,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46396 deadline: 1733740893342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:33,361 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:33,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46390 deadline: 1733740893351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:33,361 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:33,361 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:33,362 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:33,362 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:33,362 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:33,362 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:33,362 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:33,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46398 deadline: 1733740893358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:33,363 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:33,363 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:33,363 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:33,364 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/B/46508c6b9ef0453498d2ce2e61921bf1 is 50, key is test_row_0/B:col10/1733740833062/Put/seqid=0 2024-12-09T10:40:33,369 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:33,369 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:33,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46374 deadline: 1733740893362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:33,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46362 deadline: 1733740893363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:33,370 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:33,370 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:33,370 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:33,370 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:33,370 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:33,370 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:33,390 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-09T10:40:33,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:33,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. as already flushing 2024-12-09T10:40:33,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:33,392 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:40:33,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:40:33,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:40:33,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741838_1014 (size=12001) 2024-12-09T10:40:33,549 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-09T10:40:33,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:33,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. as already flushing 2024-12-09T10:40:33,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:33,550 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:40:33,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:40:33,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:40:33,573 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:33,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46396 deadline: 1733740893570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:33,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:33,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:33,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:33,575 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:33,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46390 deadline: 1733740893570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:33,576 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:33,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46398 deadline: 1733740893571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:33,577 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:33,577 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:33,577 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:33,578 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:33,578 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:33,578 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:33,585 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:33,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46374 deadline: 1733740893581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:33,586 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:33,586 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:33,586 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:33,586 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:33,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46362 deadline: 1733740893581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:33,587 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:33,587 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:33,588 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:33,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-09T10:40:33,707 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-09T10:40:33,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:33,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. as already flushing 2024-12-09T10:40:33,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:33,707 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:40:33,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:40:33,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:40:33,811 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/B/46508c6b9ef0453498d2ce2e61921bf1 2024-12-09T10:40:33,864 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-09T10:40:33,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:33,864 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/C/b0dc13c0bbab416f838e7e532d23cf8f is 50, key is test_row_0/C:col10/1733740833062/Put/seqid=0 2024-12-09T10:40:33,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. as already flushing 2024-12-09T10:40:33,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:33,864 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:40:33,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:40:33,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:40:33,882 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:33,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46396 deadline: 1733740893880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:33,884 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:33,884 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:33,884 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:33,894 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:33,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46390 deadline: 1733740893891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:33,896 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:33,896 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:33,896 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:33,896 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:33,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46398 deadline: 1733740893892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:33,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:33,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:33,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:33,897 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:33,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46374 deadline: 1733740893892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:33,900 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:33,900 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:33,900 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:33,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741839_1015 (size=12001) 2024-12-09T10:40:33,904 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:33,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46362 deadline: 1733740893900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:33,906 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:33,906 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:33,906 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:33,910 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/C/b0dc13c0bbab416f838e7e532d23cf8f 2024-12-09T10:40:33,927 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/A/0cf007a7125d4686b4d48808b19fa98b as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/0cf007a7125d4686b4d48808b19fa98b 2024-12-09T10:40:33,940 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/0cf007a7125d4686b4d48808b19fa98b, entries=150, sequenceid=14, filesize=11.7 K 2024-12-09T10:40:33,949 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/B/46508c6b9ef0453498d2ce2e61921bf1 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/46508c6b9ef0453498d2ce2e61921bf1 2024-12-09T10:40:33,961 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/46508c6b9ef0453498d2ce2e61921bf1, entries=150, sequenceid=14, filesize=11.7 K 2024-12-09T10:40:33,963 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/C/b0dc13c0bbab416f838e7e532d23cf8f as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/b0dc13c0bbab416f838e7e532d23cf8f 2024-12-09T10:40:33,976 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/b0dc13c0bbab416f838e7e532d23cf8f, entries=150, sequenceid=14, filesize=11.7 K 2024-12-09T10:40:33,986 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for 1d9d6bba7dcd09663c7b08474fc1c1b5 in 901ms, sequenceid=14, compaction requested=false 2024-12-09T10:40:33,988 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-09T10:40:33,990 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1d9d6bba7dcd09663c7b08474fc1c1b5: 2024-12-09T10:40:34,021 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-09T10:40:34,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:34,022 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 1d9d6bba7dcd09663c7b08474fc1c1b5 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-09T10:40:34,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d9d6bba7dcd09663c7b08474fc1c1b5, store=A 2024-12-09T10:40:34,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:40:34,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d9d6bba7dcd09663c7b08474fc1c1b5, store=B 2024-12-09T10:40:34,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:40:34,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d9d6bba7dcd09663c7b08474fc1c1b5, store=C 2024-12-09T10:40:34,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:40:34,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/A/a9ed01b2d09446af94cac32ee8f8212e is 50, key is test_row_0/A:col10/1733740833169/Put/seqid=0 2024-12-09T10:40:34,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741840_1016 (size=12001) 2024-12-09T10:40:34,066 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=34 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/A/a9ed01b2d09446af94cac32ee8f8212e 2024-12-09T10:40:34,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/B/6f37fb8fc8d04f47ac975c7e6d71de12 is 50, key is test_row_0/B:col10/1733740833169/Put/seqid=0 2024-12-09T10:40:34,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741841_1017 (size=12001) 2024-12-09T10:40:34,118 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=34 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/B/6f37fb8fc8d04f47ac975c7e6d71de12 2024-12-09T10:40:34,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/C/940aa690831547bb83126b96492c4837 is 50, key is test_row_0/C:col10/1733740833169/Put/seqid=0 2024-12-09T10:40:34,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-09T10:40:34,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741842_1018 (size=12001) 2024-12-09T10:40:34,193 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=34 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/C/940aa690831547bb83126b96492c4837 2024-12-09T10:40:34,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/A/a9ed01b2d09446af94cac32ee8f8212e as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/a9ed01b2d09446af94cac32ee8f8212e 2024-12-09T10:40:34,244 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/a9ed01b2d09446af94cac32ee8f8212e, entries=150, sequenceid=34, filesize=11.7 K 2024-12-09T10:40:34,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/B/6f37fb8fc8d04f47ac975c7e6d71de12 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/6f37fb8fc8d04f47ac975c7e6d71de12 2024-12-09T10:40:34,279 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/6f37fb8fc8d04f47ac975c7e6d71de12, entries=150, sequenceid=34, filesize=11.7 K 2024-12-09T10:40:34,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/C/940aa690831547bb83126b96492c4837 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/940aa690831547bb83126b96492c4837 2024-12-09T10:40:34,307 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/940aa690831547bb83126b96492c4837, entries=150, sequenceid=34, filesize=11.7 K 2024-12-09T10:40:34,310 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=0 B/0 for 1d9d6bba7dcd09663c7b08474fc1c1b5 in 289ms, sequenceid=34, compaction requested=false 2024-12-09T10:40:34,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 1d9d6bba7dcd09663c7b08474fc1c1b5: 2024-12-09T10:40:34,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:34,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-12-09T10:40:34,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-12-09T10:40:34,317 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-09T10:40:34,317 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2900 sec 2024-12-09T10:40:34,323 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 1.3260 sec 2024-12-09T10:40:34,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(8855): Flush requested on 1d9d6bba7dcd09663c7b08474fc1c1b5 2024-12-09T10:40:34,425 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1d9d6bba7dcd09663c7b08474fc1c1b5 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-09T10:40:34,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d9d6bba7dcd09663c7b08474fc1c1b5, store=A 2024-12-09T10:40:34,429 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:40:34,429 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d9d6bba7dcd09663c7b08474fc1c1b5, store=B 2024-12-09T10:40:34,429 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:40:34,429 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d9d6bba7dcd09663c7b08474fc1c1b5, store=C 2024-12-09T10:40:34,429 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:40:34,441 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/A/4b4ce92d705b4485b69a61cf69095a0f is 50, key is test_row_0/A:col10/1733740834415/Put/seqid=0 2024-12-09T10:40:34,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741843_1019 (size=14341) 2024-12-09T10:40:34,454 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=45 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/A/4b4ce92d705b4485b69a61cf69095a0f 2024-12-09T10:40:34,476 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/B/e503f95ccd714dbc8594dccdad97f8a2 is 50, key is test_row_0/B:col10/1733740834415/Put/seqid=0 2024-12-09T10:40:34,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741844_1020 (size=12001) 2024-12-09T10:40:34,490 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=45 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/B/e503f95ccd714dbc8594dccdad97f8a2 2024-12-09T10:40:34,508 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/C/4d3390a1629343ea906cc28c880e521c is 50, key is test_row_0/C:col10/1733740834415/Put/seqid=0 2024-12-09T10:40:34,517 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733740834487, max timestamp=1733740834506Num uniques 859; 2024-12-09T10:40:34,517 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733740834487, max timestamp=1733740834507Num uniques 859; 2024-12-09T10:40:34,520 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T10:40:34,522 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T10:40:34,534 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733740834487, max timestamp=1733740834506Num uniques 859; 2024-12-09T10:40:34,534 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-09T10:40:34,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741845_1021 (size=12001) 2024-12-09T10:40:34,541 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=45 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/C/4d3390a1629343ea906cc28c880e521c 2024-12-09T10:40:34,555 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/A/4b4ce92d705b4485b69a61cf69095a0f as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/4b4ce92d705b4485b69a61cf69095a0f 2024-12-09T10:40:34,569 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/4b4ce92d705b4485b69a61cf69095a0f, entries=200, sequenceid=45, filesize=14.0 K 2024-12-09T10:40:34,573 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/B/e503f95ccd714dbc8594dccdad97f8a2 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/e503f95ccd714dbc8594dccdad97f8a2 2024-12-09T10:40:34,590 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/e503f95ccd714dbc8594dccdad97f8a2, entries=150, sequenceid=45, filesize=11.7 K 2024-12-09T10:40:34,593 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/C/4d3390a1629343ea906cc28c880e521c as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/4d3390a1629343ea906cc28c880e521c 2024-12-09T10:40:34,597 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:34,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46362 deadline: 1733740894575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:34,599 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:34,599 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:34,599 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:34,599 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:34,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46396 deadline: 1733740894576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:34,600 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:34,600 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:34,600 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:34,601 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:34,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46390 deadline: 1733740894578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:34,602 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:34,602 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:34,602 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:34,602 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:34,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46398 deadline: 1733740894592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:34,604 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:34,604 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:34,604 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:34,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:34,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46374 deadline: 1733740894597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:34,606 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:34,606 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:34,607 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:34,607 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/4d3390a1629343ea906cc28c880e521c, entries=150, sequenceid=45, filesize=11.7 K 2024-12-09T10:40:34,610 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.08 KB/130125 for 1d9d6bba7dcd09663c7b08474fc1c1b5 in 184ms, sequenceid=45, compaction requested=true 2024-12-09T10:40:34,610 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1d9d6bba7dcd09663c7b08474fc1c1b5: 2024-12-09T10:40:34,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1d9d6bba7dcd09663c7b08474fc1c1b5:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T10:40:34,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:40:34,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1d9d6bba7dcd09663c7b08474fc1c1b5:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T10:40:34,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:40:34,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1d9d6bba7dcd09663c7b08474fc1c1b5:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T10:40:34,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T10:40:34,621 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T10:40:34,622 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T10:40:34,626 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T10:40:34,626 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T10:40:34,628 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1541): 1d9d6bba7dcd09663c7b08474fc1c1b5/B is initiating minor compaction (all files) 2024-12-09T10:40:34,629 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1d9d6bba7dcd09663c7b08474fc1c1b5/B in TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:34,629 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/46508c6b9ef0453498d2ce2e61921bf1, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/6f37fb8fc8d04f47ac975c7e6d71de12, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/e503f95ccd714dbc8594dccdad97f8a2] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp, totalSize=35.2 K 2024-12-09T10:40:34,631 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 46508c6b9ef0453498d2ce2e61921bf1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733740833062 2024-12-09T10:40:34,632 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f37fb8fc8d04f47ac975c7e6d71de12, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=34, earliestPutTs=1733740833169 2024-12-09T10:40:34,633 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting e503f95ccd714dbc8594dccdad97f8a2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=45, earliestPutTs=1733740834415 2024-12-09T10:40:34,630 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1541): 1d9d6bba7dcd09663c7b08474fc1c1b5/A is initiating minor compaction (all files) 2024-12-09T10:40:34,639 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1d9d6bba7dcd09663c7b08474fc1c1b5/A in TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:34,639 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/0cf007a7125d4686b4d48808b19fa98b, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/a9ed01b2d09446af94cac32ee8f8212e, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/4b4ce92d705b4485b69a61cf69095a0f] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp, totalSize=37.4 K 2024-12-09T10:40:34,642 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0cf007a7125d4686b4d48808b19fa98b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733740833062 2024-12-09T10:40:34,643 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting a9ed01b2d09446af94cac32ee8f8212e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=34, earliestPutTs=1733740833169 2024-12-09T10:40:34,644 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4b4ce92d705b4485b69a61cf69095a0f, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=45, earliestPutTs=1733740834415 2024-12-09T10:40:34,667 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1d9d6bba7dcd09663c7b08474fc1c1b5#B#compaction#9 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:40:34,668 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/B/8540a1ed92f847edb33d38ffa69f8ffe is 50, key is test_row_0/B:col10/1733740834415/Put/seqid=0 2024-12-09T10:40:34,679 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1d9d6bba7dcd09663c7b08474fc1c1b5#A#compaction#10 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:40:34,680 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/A/32fe83a127de402ea134cac0a94c1b3e is 50, key is test_row_0/A:col10/1733740834415/Put/seqid=0 2024-12-09T10:40:34,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741846_1022 (size=12104) 2024-12-09T10:40:34,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741847_1023 (size=12104) 2024-12-09T10:40:34,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(8855): Flush requested on 1d9d6bba7dcd09663c7b08474fc1c1b5 2024-12-09T10:40:34,724 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1d9d6bba7dcd09663c7b08474fc1c1b5 3/3 column families, dataSize=133.78 KB heapSize=384.07 KB 2024-12-09T10:40:34,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d9d6bba7dcd09663c7b08474fc1c1b5, store=A 2024-12-09T10:40:34,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T10:40:34,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d9d6bba7dcd09663c7b08474fc1c1b5, store=B 2024-12-09T10:40:34,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T10:40:34,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d9d6bba7dcd09663c7b08474fc1c1b5, store=C 2024-12-09T10:40:34,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T10:40:34,744 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/A/996018c21d024fffb1aa2bc5adb98387 is 50, key is test_row_0/A:col10/1733740834719/Put/seqid=0 2024-12-09T10:40:34,748 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/B/8540a1ed92f847edb33d38ffa69f8ffe as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/8540a1ed92f847edb33d38ffa69f8ffe 2024-12-09T10:40:34,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741848_1024 (size=14341) 2024-12-09T10:40:34,820 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:34,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46374 deadline: 1733740894740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:34,824 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1d9d6bba7dcd09663c7b08474fc1c1b5/B of 1d9d6bba7dcd09663c7b08474fc1c1b5 into 8540a1ed92f847edb33d38ffa69f8ffe(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:40:34,824 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1d9d6bba7dcd09663c7b08474fc1c1b5: 2024-12-09T10:40:34,824 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:34,824 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:34,825 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:34,827 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., storeName=1d9d6bba7dcd09663c7b08474fc1c1b5/B, priority=13, startTime=1733740834621; duration=0sec 2024-12-09T10:40:34,828 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T10:40:34,828 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1d9d6bba7dcd09663c7b08474fc1c1b5:B 2024-12-09T10:40:34,828 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T10:40:34,831 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T10:40:34,832 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:34,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46362 deadline: 1733740894752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:34,833 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1541): 1d9d6bba7dcd09663c7b08474fc1c1b5/C is initiating minor compaction (all files) 2024-12-09T10:40:34,833 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1d9d6bba7dcd09663c7b08474fc1c1b5/C in TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:34,833 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/b0dc13c0bbab416f838e7e532d23cf8f, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/940aa690831547bb83126b96492c4837, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/4d3390a1629343ea906cc28c880e521c] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp, totalSize=35.2 K 2024-12-09T10:40:34,834 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:34,834 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:34,834 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:34,835 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting b0dc13c0bbab416f838e7e532d23cf8f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733740833062 2024-12-09T10:40:34,835 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 940aa690831547bb83126b96492c4837, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=34, earliestPutTs=1733740833169 2024-12-09T10:40:34,836 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d3390a1629343ea906cc28c880e521c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=45, earliestPutTs=1733740834415 2024-12-09T10:40:34,836 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:34,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46390 deadline: 1733740894821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:34,838 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:34,838 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:34,839 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:34,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:34,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46398 deadline: 1733740894822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:34,840 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:34,840 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:34,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:34,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:34,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46396 deadline: 1733740894823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:34,845 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:34,845 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:34,845 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:34,863 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1d9d6bba7dcd09663c7b08474fc1c1b5#C#compaction#12 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:40:34,864 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/C/d7dea30f31c2466e8eb0c15ec2452db0 is 50, key is test_row_0/C:col10/1733740834415/Put/seqid=0 2024-12-09T10:40:34,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741849_1025 (size=12104) 2024-12-09T10:40:34,901 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/C/d7dea30f31c2466e8eb0c15ec2452db0 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/d7dea30f31c2466e8eb0c15ec2452db0 2024-12-09T10:40:34,914 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1d9d6bba7dcd09663c7b08474fc1c1b5/C of 1d9d6bba7dcd09663c7b08474fc1c1b5 into d7dea30f31c2466e8eb0c15ec2452db0(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:40:34,914 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1d9d6bba7dcd09663c7b08474fc1c1b5: 2024-12-09T10:40:34,914 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., storeName=1d9d6bba7dcd09663c7b08474fc1c1b5/C, priority=13, startTime=1733740834621; duration=0sec 2024-12-09T10:40:34,915 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:40:34,915 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1d9d6bba7dcd09663c7b08474fc1c1b5:C 2024-12-09T10:40:34,931 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:34,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46374 deadline: 1733740894931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:34,933 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:34,933 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:34,933 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:34,944 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:34,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46362 deadline: 1733740894940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:34,946 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:34,946 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:34,946 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:34,952 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:34,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46390 deadline: 1733740894950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:34,954 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:34,954 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:34,954 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:34,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46398 deadline: 1733740894950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:34,954 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:34,955 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:34,955 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:34,955 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:34,962 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:34,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46396 deadline: 1733740894951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:34,964 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:34,964 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:34,964 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:35,119 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/A/32fe83a127de402ea134cac0a94c1b3e as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/32fe83a127de402ea134cac0a94c1b3e 2024-12-09T10:40:35,135 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1d9d6bba7dcd09663c7b08474fc1c1b5/A of 1d9d6bba7dcd09663c7b08474fc1c1b5 into 32fe83a127de402ea134cac0a94c1b3e(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:40:35,135 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1d9d6bba7dcd09663c7b08474fc1c1b5: 2024-12-09T10:40:35,135 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., storeName=1d9d6bba7dcd09663c7b08474fc1c1b5/A, priority=13, startTime=1733740834612; duration=0sec 2024-12-09T10:40:35,136 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:40:35,136 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1d9d6bba7dcd09663c7b08474fc1c1b5:A 2024-12-09T10:40:35,146 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:35,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46374 deadline: 1733740895141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:35,147 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:35,147 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:35,148 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:35,153 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:35,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46362 deadline: 1733740895150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:35,155 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:35,155 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:35,155 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:35,168 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:35,168 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:35,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46390 deadline: 1733740895160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:35,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46398 deadline: 1733740895161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:35,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-09T10:40:35,170 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:35,170 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:35,170 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:35,170 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T10:40:35,170 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:35,171 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:35,171 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:35,171 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-09T10:40:35,171 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:35,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46396 deadline: 1733740895170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:35,173 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:35,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-12-09T10:40:35,173 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/A/996018c21d024fffb1aa2bc5adb98387 2024-12-09T10:40:35,173 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:35,173 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:35,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-09T10:40:35,175 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T10:40:35,178 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T10:40:35,178 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T10:40:35,215 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/B/9a8a4cc395ba48c092c7aa6ad8e215ae is 50, key is test_row_0/B:col10/1733740834719/Put/seqid=0 2024-12-09T10:40:35,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741850_1026 (size=12001) 2024-12-09T10:40:35,251 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/B/9a8a4cc395ba48c092c7aa6ad8e215ae 2024-12-09T10:40:35,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-09T10:40:35,281 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/C/a56e848a420b4a5cb42b81474f83fb63 is 50, key is test_row_0/C:col10/1733740834719/Put/seqid=0 2024-12-09T10:40:35,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741851_1027 (size=12001) 2024-12-09T10:40:35,325 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.33 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/C/a56e848a420b4a5cb42b81474f83fb63 2024-12-09T10:40:35,333 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-09T10:40:35,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:35,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. as already flushing 2024-12-09T10:40:35,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:35,334 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:40:35,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:40:35,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:40:35,348 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/A/996018c21d024fffb1aa2bc5adb98387 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/996018c21d024fffb1aa2bc5adb98387 2024-12-09T10:40:35,366 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/996018c21d024fffb1aa2bc5adb98387, entries=200, sequenceid=82, filesize=14.0 K 2024-12-09T10:40:35,369 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/B/9a8a4cc395ba48c092c7aa6ad8e215ae as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/9a8a4cc395ba48c092c7aa6ad8e215ae 2024-12-09T10:40:35,386 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/9a8a4cc395ba48c092c7aa6ad8e215ae, entries=150, sequenceid=82, filesize=11.7 K 2024-12-09T10:40:35,388 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/C/a56e848a420b4a5cb42b81474f83fb63 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/a56e848a420b4a5cb42b81474f83fb63 2024-12-09T10:40:35,406 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/a56e848a420b4a5cb42b81474f83fb63, entries=150, sequenceid=82, filesize=11.7 K 2024-12-09T10:40:35,408 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~133.78 KB/136995, heapSize ~384.02 KB/393240, currentSize=53.67 KB/54960 for 1d9d6bba7dcd09663c7b08474fc1c1b5 in 685ms, sequenceid=82, compaction requested=false 2024-12-09T10:40:35,409 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1d9d6bba7dcd09663c7b08474fc1c1b5: 2024-12-09T10:40:35,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(8855): Flush requested on 1d9d6bba7dcd09663c7b08474fc1c1b5 2024-12-09T10:40:35,470 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1d9d6bba7dcd09663c7b08474fc1c1b5 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-09T10:40:35,479 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d9d6bba7dcd09663c7b08474fc1c1b5, store=A 2024-12-09T10:40:35,479 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:40:35,479 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d9d6bba7dcd09663c7b08474fc1c1b5, store=B 2024-12-09T10:40:35,479 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:40:35,479 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d9d6bba7dcd09663c7b08474fc1c1b5, store=C 2024-12-09T10:40:35,479 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:40:35,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-09T10:40:35,491 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-09T10:40:35,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:35,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. as already flushing 2024-12-09T10:40:35,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:35,492 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:40:35,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:40:35,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:40:35,503 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/A/fd326f8fa2024083ab10771dc379cacd is 50, key is test_row_0/A:col10/1733740834751/Put/seqid=0 2024-12-09T10:40:35,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741852_1028 (size=14341) 2024-12-09T10:40:35,547 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/A/fd326f8fa2024083ab10771dc379cacd 2024-12-09T10:40:35,549 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:35,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46390 deadline: 1733740895535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:35,550 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:35,550 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:35,550 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:35,550 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:35,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46398 deadline: 1733740895537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:35,551 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:35,551 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:35,552 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:35,556 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:35,556 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:35,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46396 deadline: 1733740895543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:35,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46374 deadline: 1733740895546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:35,558 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:35,558 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:35,558 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:35,558 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:35,558 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:35,558 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:35,558 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:35,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46362 deadline: 1733740895550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:35,559 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:35,559 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:35,559 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:35,580 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/B/59fa03b684d14f508ad8bd15f9079456 is 50, key is test_row_0/B:col10/1733740834751/Put/seqid=0 2024-12-09T10:40:35,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741853_1029 (size=12001) 2024-12-09T10:40:35,655 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-09T10:40:35,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:35,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. as already flushing 2024-12-09T10:40:35,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:35,656 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:40:35,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:40:35,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:40:35,665 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:35,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46390 deadline: 1733740895661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:35,667 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:35,668 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:35,668 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:35,673 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:35,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46374 deadline: 1733740895666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:35,674 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:35,674 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:35,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46398 deadline: 1733740895666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:35,674 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:35,674 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:35,675 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:35,675 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:35,675 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:35,676 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:35,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46362 deadline: 1733740895670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:35,677 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:35,677 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:35,677 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:35,677 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:35,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46396 deadline: 1733740895670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:35,678 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:35,678 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:35,678 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:35,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-09T10:40:35,819 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-09T10:40:35,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:35,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. as already flushing 2024-12-09T10:40:35,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:35,819 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:40:35,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:40:35,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:40:35,884 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:35,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46374 deadline: 1733740895880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:35,885 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:35,885 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:35,885 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:35,885 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:35,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46398 deadline: 1733740895880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:35,886 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:35,886 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:35,886 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:35,886 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:35,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46390 deadline: 1733740895881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:35,887 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:35,887 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:35,887 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:35,892 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:35,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46362 deadline: 1733740895889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:35,893 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:35,893 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:35,893 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:35,893 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:35,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46396 deadline: 1733740895890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:35,894 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:35,894 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:35,894 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:35,974 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-09T10:40:35,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:35,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. as already flushing 2024-12-09T10:40:35,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:35,975 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:40:35,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:40:35,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:40:36,014 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/B/59fa03b684d14f508ad8bd15f9079456 2024-12-09T10:40:36,037 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/C/9549ec226fb14a8c80e95872ee6ff28c is 50, key is test_row_0/C:col10/1733740834751/Put/seqid=0 2024-12-09T10:40:36,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741854_1030 (size=12001) 2024-12-09T10:40:36,057 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/C/9549ec226fb14a8c80e95872ee6ff28c 2024-12-09T10:40:36,074 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/A/fd326f8fa2024083ab10771dc379cacd as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/fd326f8fa2024083ab10771dc379cacd 2024-12-09T10:40:36,092 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/fd326f8fa2024083ab10771dc379cacd, entries=200, sequenceid=98, filesize=14.0 K 2024-12-09T10:40:36,094 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/B/59fa03b684d14f508ad8bd15f9079456 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/59fa03b684d14f508ad8bd15f9079456 2024-12-09T10:40:36,113 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/59fa03b684d14f508ad8bd15f9079456, entries=150, sequenceid=98, filesize=11.7 K 2024-12-09T10:40:36,126 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/C/9549ec226fb14a8c80e95872ee6ff28c as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/9549ec226fb14a8c80e95872ee6ff28c 2024-12-09T10:40:36,131 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-09T10:40:36,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:36,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. as already flushing 2024-12-09T10:40:36,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:36,131 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:40:36,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:40:36,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:40:36,152 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/9549ec226fb14a8c80e95872ee6ff28c, entries=150, sequenceid=98, filesize=11.7 K 2024-12-09T10:40:36,156 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 1d9d6bba7dcd09663c7b08474fc1c1b5 in 685ms, sequenceid=98, compaction requested=true 2024-12-09T10:40:36,156 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1d9d6bba7dcd09663c7b08474fc1c1b5: 2024-12-09T10:40:36,156 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1d9d6bba7dcd09663c7b08474fc1c1b5:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T10:40:36,156 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:40:36,156 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T10:40:36,156 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1d9d6bba7dcd09663c7b08474fc1c1b5:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T10:40:36,156 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:40:36,156 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T10:40:36,156 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1d9d6bba7dcd09663c7b08474fc1c1b5:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T10:40:36,156 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T10:40:36,159 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T10:40:36,159 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1541): 1d9d6bba7dcd09663c7b08474fc1c1b5/B is initiating minor compaction (all files) 2024-12-09T10:40:36,160 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1d9d6bba7dcd09663c7b08474fc1c1b5/B in TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:36,160 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/8540a1ed92f847edb33d38ffa69f8ffe, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/9a8a4cc395ba48c092c7aa6ad8e215ae, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/59fa03b684d14f508ad8bd15f9079456] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp, totalSize=35.3 K 2024-12-09T10:40:36,160 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40786 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T10:40:36,160 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1541): 1d9d6bba7dcd09663c7b08474fc1c1b5/A is initiating minor compaction (all files) 2024-12-09T10:40:36,160 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1d9d6bba7dcd09663c7b08474fc1c1b5/A in TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:36,161 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/32fe83a127de402ea134cac0a94c1b3e, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/996018c21d024fffb1aa2bc5adb98387, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/fd326f8fa2024083ab10771dc379cacd] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp, totalSize=39.8 K 2024-12-09T10:40:36,161 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 8540a1ed92f847edb33d38ffa69f8ffe, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=45, earliestPutTs=1733740834415 2024-12-09T10:40:36,161 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 32fe83a127de402ea134cac0a94c1b3e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=45, earliestPutTs=1733740834415 2024-12-09T10:40:36,163 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 996018c21d024fffb1aa2bc5adb98387, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1733740834552 2024-12-09T10:40:36,163 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting fd326f8fa2024083ab10771dc379cacd, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1733740834751 2024-12-09T10:40:36,164 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a8a4cc395ba48c092c7aa6ad8e215ae, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1733740834552 2024-12-09T10:40:36,164 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-09T10:40:36,169 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 59fa03b684d14f508ad8bd15f9079456, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1733740834751 2024-12-09T10:40:36,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(8855): Flush requested on 1d9d6bba7dcd09663c7b08474fc1c1b5 2024-12-09T10:40:36,194 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1d9d6bba7dcd09663c7b08474fc1c1b5 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-09T10:40:36,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d9d6bba7dcd09663c7b08474fc1c1b5, store=A 2024-12-09T10:40:36,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:40:36,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d9d6bba7dcd09663c7b08474fc1c1b5, store=B 2024-12-09T10:40:36,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:40:36,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d9d6bba7dcd09663c7b08474fc1c1b5, store=C 2024-12-09T10:40:36,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:40:36,202 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1d9d6bba7dcd09663c7b08474fc1c1b5#B#compaction#18 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:40:36,203 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/B/a31be429598e44d0a5b6399d21228c95 is 50, key is test_row_0/B:col10/1733740834751/Put/seqid=0 2024-12-09T10:40:36,210 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1d9d6bba7dcd09663c7b08474fc1c1b5#A#compaction#19 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:40:36,211 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/A/519baef860a141f2b3ec677210dad12d is 50, key is test_row_0/A:col10/1733740834751/Put/seqid=0 2024-12-09T10:40:36,215 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/A/d561b4afe4e14addb37708472b2ea07e is 50, key is test_row_0/A:col10/1733740835531/Put/seqid=0 2024-12-09T10:40:36,243 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:36,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46396 deadline: 1733740896228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:36,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:36,244 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:36,245 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:36,245 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:36,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46374 deadline: 1733740896231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:36,246 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:36,246 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:36,246 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:36,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:36,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46398 deadline: 1733740896239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:36,253 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:36,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46390 deadline: 1733740896242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:36,253 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:36,254 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:36,254 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:36,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46362 deadline: 1733740896244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:36,254 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:36,254 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:36,254 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:36,254 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:36,255 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:36,255 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:36,255 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:36,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741856_1032 (size=12207) 2024-12-09T10:40:36,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741855_1031 (size=12207) 2024-12-09T10:40:36,291 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-09T10:40:36,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:36,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. as already flushing 2024-12-09T10:40:36,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:36,292 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:40:36,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:40:36,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:40:36,295 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/B/a31be429598e44d0a5b6399d21228c95 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/a31be429598e44d0a5b6399d21228c95 2024-12-09T10:40:36,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741857_1033 (size=14341) 2024-12-09T10:40:36,305 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/A/d561b4afe4e14addb37708472b2ea07e 2024-12-09T10:40:36,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-09T10:40:36,313 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1d9d6bba7dcd09663c7b08474fc1c1b5/B of 1d9d6bba7dcd09663c7b08474fc1c1b5 into a31be429598e44d0a5b6399d21228c95(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:40:36,315 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1d9d6bba7dcd09663c7b08474fc1c1b5: 2024-12-09T10:40:36,316 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., storeName=1d9d6bba7dcd09663c7b08474fc1c1b5/B, priority=13, startTime=1733740836156; duration=0sec 2024-12-09T10:40:36,316 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T10:40:36,316 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1d9d6bba7dcd09663c7b08474fc1c1b5:B 2024-12-09T10:40:36,316 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T10:40:36,319 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T10:40:36,319 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1541): 1d9d6bba7dcd09663c7b08474fc1c1b5/C is initiating minor compaction (all files) 2024-12-09T10:40:36,320 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1d9d6bba7dcd09663c7b08474fc1c1b5/C in TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:36,320 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/d7dea30f31c2466e8eb0c15ec2452db0, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/a56e848a420b4a5cb42b81474f83fb63, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/9549ec226fb14a8c80e95872ee6ff28c] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp, totalSize=35.3 K 2024-12-09T10:40:36,321 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting d7dea30f31c2466e8eb0c15ec2452db0, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=45, earliestPutTs=1733740834415 2024-12-09T10:40:36,322 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting a56e848a420b4a5cb42b81474f83fb63, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1733740834552 2024-12-09T10:40:36,323 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 9549ec226fb14a8c80e95872ee6ff28c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1733740834751 2024-12-09T10:40:36,345 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/B/68dfe148d84d4c1a9f484a389dd84455 is 50, key is test_row_0/B:col10/1733740835531/Put/seqid=0 2024-12-09T10:40:36,350 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:36,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46396 deadline: 1733740896350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:36,351 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:36,351 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:36,352 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:36,352 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:36,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46374 deadline: 1733740896350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:36,352 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:36,353 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:36,353 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:36,358 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1d9d6bba7dcd09663c7b08474fc1c1b5#C#compaction#22 average throughput is 0.47 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:40:36,359 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/C/6a86fc708dfb4b99926cd2728a25f1d5 is 50, key is test_row_0/C:col10/1733740834751/Put/seqid=0 2024-12-09T10:40:36,364 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:36,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46390 deadline: 1733740896359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:36,365 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:36,365 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:36,365 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:36,365 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:36,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46398 deadline: 1733740896360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:36,366 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:36,366 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:36,366 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:36,366 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:36,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46362 deadline: 1733740896360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:36,367 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:36,367 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:36,367 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:36,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741858_1034 (size=12001) 2024-12-09T10:40:36,383 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/B/68dfe148d84d4c1a9f484a389dd84455 2024-12-09T10:40:36,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741859_1035 (size=12207) 2024-12-09T10:40:36,423 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/C/ccd5118581644949a62cd9f2b852f010 is 50, key is test_row_0/C:col10/1733740835531/Put/seqid=0 2024-12-09T10:40:36,427 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/C/6a86fc708dfb4b99926cd2728a25f1d5 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/6a86fc708dfb4b99926cd2728a25f1d5 2024-12-09T10:40:36,440 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1d9d6bba7dcd09663c7b08474fc1c1b5/C of 1d9d6bba7dcd09663c7b08474fc1c1b5 into 6a86fc708dfb4b99926cd2728a25f1d5(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:40:36,440 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1d9d6bba7dcd09663c7b08474fc1c1b5: 2024-12-09T10:40:36,440 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., storeName=1d9d6bba7dcd09663c7b08474fc1c1b5/C, priority=13, startTime=1733740836156; duration=0sec 2024-12-09T10:40:36,440 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:40:36,440 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1d9d6bba7dcd09663c7b08474fc1c1b5:C 2024-12-09T10:40:36,448 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-09T10:40:36,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:36,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. as already flushing 2024-12-09T10:40:36,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:36,448 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:40:36,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:40:36,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:40:36,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741860_1036 (size=12001) 2024-12-09T10:40:36,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:36,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46396 deadline: 1733740896561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:36,565 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:36,565 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:36,565 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:36,565 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:36,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46374 deadline: 1733740896561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:36,566 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:36,566 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:36,566 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:36,571 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:36,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46362 deadline: 1733740896570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:36,572 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:36,572 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:36,572 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:36,573 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:36,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46398 deadline: 1733740896570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:36,574 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:36,574 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:36,574 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:36,577 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:36,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46390 deadline: 1733740896572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:36,578 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:36,578 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:36,578 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:36,604 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-09T10:40:36,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:36,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. as already flushing 2024-12-09T10:40:36,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:36,604 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:40:36,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:40:36,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:40:36,686 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/A/519baef860a141f2b3ec677210dad12d as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/519baef860a141f2b3ec677210dad12d 2024-12-09T10:40:36,709 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1d9d6bba7dcd09663c7b08474fc1c1b5/A of 1d9d6bba7dcd09663c7b08474fc1c1b5 into 519baef860a141f2b3ec677210dad12d(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:40:36,709 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1d9d6bba7dcd09663c7b08474fc1c1b5: 2024-12-09T10:40:36,709 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., storeName=1d9d6bba7dcd09663c7b08474fc1c1b5/A, priority=13, startTime=1733740836156; duration=0sec 2024-12-09T10:40:36,710 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:40:36,710 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1d9d6bba7dcd09663c7b08474fc1c1b5:A 2024-12-09T10:40:36,759 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-09T10:40:36,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:36,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. as already flushing 2024-12-09T10:40:36,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:36,760 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:40:36,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:40:36,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:40:36,817 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-09T10:40:36,870 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/C/ccd5118581644949a62cd9f2b852f010 2024-12-09T10:40:36,878 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:36,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46396 deadline: 1733740896870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:36,880 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:36,880 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:36,880 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:36,883 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/A/d561b4afe4e14addb37708472b2ea07e as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/d561b4afe4e14addb37708472b2ea07e 2024-12-09T10:40:36,884 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:36,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46362 deadline: 1733740896880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:36,885 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:36,885 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:36,885 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:36,885 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:36,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46398 deadline: 1733740896880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:36,886 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:36,886 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:36,886 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:36,886 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:36,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46374 deadline: 1733740896881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:36,887 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:36,887 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:36,887 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:36,897 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:36,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46390 deadline: 1733740896890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:36,899 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:36,899 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:36,899 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:36,903 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/d561b4afe4e14addb37708472b2ea07e, entries=200, sequenceid=118, filesize=14.0 K 2024-12-09T10:40:36,906 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/B/68dfe148d84d4c1a9f484a389dd84455 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/68dfe148d84d4c1a9f484a389dd84455 2024-12-09T10:40:36,917 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/68dfe148d84d4c1a9f484a389dd84455, entries=150, sequenceid=118, filesize=11.7 K 2024-12-09T10:40:36,919 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/C/ccd5118581644949a62cd9f2b852f010 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/ccd5118581644949a62cd9f2b852f010 2024-12-09T10:40:36,925 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-09T10:40:36,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:36,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. as already flushing 2024-12-09T10:40:36,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:36,927 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:40:36,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:40:36,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:40:36,935 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/ccd5118581644949a62cd9f2b852f010, entries=150, sequenceid=118, filesize=11.7 K 2024-12-09T10:40:36,937 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 1d9d6bba7dcd09663c7b08474fc1c1b5 in 743ms, sequenceid=118, compaction requested=false 2024-12-09T10:40:36,938 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1d9d6bba7dcd09663c7b08474fc1c1b5: 2024-12-09T10:40:37,082 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-09T10:40:37,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:37,083 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 1d9d6bba7dcd09663c7b08474fc1c1b5 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-09T10:40:37,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d9d6bba7dcd09663c7b08474fc1c1b5, store=A 2024-12-09T10:40:37,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:40:37,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d9d6bba7dcd09663c7b08474fc1c1b5, store=B 2024-12-09T10:40:37,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:40:37,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d9d6bba7dcd09663c7b08474fc1c1b5, store=C 2024-12-09T10:40:37,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:40:37,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/A/4ff24538ff4e428288740e499ae36eca is 50, key is test_row_0/A:col10/1733740836215/Put/seqid=0 2024-12-09T10:40:37,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741861_1037 (size=12051) 2024-12-09T10:40:37,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-09T10:40:37,344 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-09T10:40:37,344 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-09T10:40:37,349 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-09T10:40:37,349 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-09T10:40:37,350 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-09T10:40:37,350 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-09T10:40:37,352 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-09T10:40:37,352 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-12-09T10:40:37,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(8855): Flush requested on 1d9d6bba7dcd09663c7b08474fc1c1b5 2024-12-09T10:40:37,396 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. as already flushing 2024-12-09T10:40:37,438 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:37,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46362 deadline: 1733740897430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:37,439 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:37,440 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:37,440 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:37,440 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733740837415, max timestamp=1733740837433Num uniques 859; 2024-12-09T10:40:37,440 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T10:40:37,460 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733740837415, max timestamp=1733740837433Num uniques 859; 2024-12-09T10:40:37,460 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T10:40:37,464 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733740837415, max timestamp=1733740837433Num uniques 859; 2024-12-09T10:40:37,464 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T10:40:37,494 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:37,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46396 deadline: 1733740897482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:37,495 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:37,495 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:37,495 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:37,499 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:37,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46398 deadline: 1733740897492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:37,500 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:37,500 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:37,500 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:37,501 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:37,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46390 deadline: 1733740897495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:37,502 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:37,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46374 deadline: 1733740897495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:37,502 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:37,502 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:37,502 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:37,503 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:37,503 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:37,503 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:37,538 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/A/4ff24538ff4e428288740e499ae36eca 2024-12-09T10:40:37,555 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:37,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/B/dc59d33a81a9401c807c211463f76a9a is 50, key is test_row_0/B:col10/1733740836215/Put/seqid=0 2024-12-09T10:40:37,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46362 deadline: 1733740897550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:37,556 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:37,556 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:37,557 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:37,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741862_1038 (size=12051) 2024-12-09T10:40:37,579 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/B/dc59d33a81a9401c807c211463f76a9a 2024-12-09T10:40:37,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/C/307768823c2544879488b5dd67fe12c6 is 50, key is test_row_0/C:col10/1733740836215/Put/seqid=0 2024-12-09T10:40:37,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:37,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46396 deadline: 1733740897601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:37,606 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:37,606 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:37,606 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:37,615 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:37,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46390 deadline: 1733740897610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:37,616 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:37,616 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:37,617 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:37,617 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:37,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46398 deadline: 1733740897610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:37,617 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:37,617 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:37,617 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:37,617 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:37,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46374 deadline: 1733740897611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:37,618 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:37,618 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:37,618 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:37,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741863_1039 (size=12051) 2024-12-09T10:40:37,631 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/C/307768823c2544879488b5dd67fe12c6 2024-12-09T10:40:37,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/A/4ff24538ff4e428288740e499ae36eca as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/4ff24538ff4e428288740e499ae36eca 2024-12-09T10:40:37,657 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/4ff24538ff4e428288740e499ae36eca, entries=150, sequenceid=133, filesize=11.8 K 2024-12-09T10:40:37,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/B/dc59d33a81a9401c807c211463f76a9a as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/dc59d33a81a9401c807c211463f76a9a 2024-12-09T10:40:37,672 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/dc59d33a81a9401c807c211463f76a9a, entries=150, sequenceid=133, filesize=11.8 K 2024-12-09T10:40:37,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/C/307768823c2544879488b5dd67fe12c6 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/307768823c2544879488b5dd67fe12c6 2024-12-09T10:40:37,688 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/307768823c2544879488b5dd67fe12c6, entries=150, sequenceid=133, filesize=11.8 K 2024-12-09T10:40:37,690 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=129.71 KB/132820 for 1d9d6bba7dcd09663c7b08474fc1c1b5 in 607ms, sequenceid=133, compaction requested=true 2024-12-09T10:40:37,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 1d9d6bba7dcd09663c7b08474fc1c1b5: 2024-12-09T10:40:37,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:37,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-12-09T10:40:37,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-12-09T10:40:37,700 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-12-09T10:40:37,700 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5190 sec 2024-12-09T10:40:37,702 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 2.5300 sec 2024-12-09T10:40:37,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(8855): Flush requested on 1d9d6bba7dcd09663c7b08474fc1c1b5 2024-12-09T10:40:37,765 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1d9d6bba7dcd09663c7b08474fc1c1b5 3/3 column families, dataSize=136.42 KB heapSize=390.06 KB 2024-12-09T10:40:37,765 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d9d6bba7dcd09663c7b08474fc1c1b5, store=A 2024-12-09T10:40:37,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T10:40:37,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d9d6bba7dcd09663c7b08474fc1c1b5, store=B 2024-12-09T10:40:37,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T10:40:37,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d9d6bba7dcd09663c7b08474fc1c1b5, store=C 2024-12-09T10:40:37,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T10:40:37,776 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/A/84f13aafe378489c826b7afabf6b0dd4 is 50, key is test_row_0/A:col10/1733740837492/Put/seqid=0 2024-12-09T10:40:37,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741864_1040 (size=14541) 2024-12-09T10:40:37,808 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/A/84f13aafe378489c826b7afabf6b0dd4 2024-12-09T10:40:37,825 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:37,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46390 deadline: 1733740897820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:37,826 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:37,826 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:37,826 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:37,826 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:37,826 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/B/2f73e1188d8b4a598b576b535fa20f34 is 50, key is test_row_0/B:col10/1733740837492/Put/seqid=0 2024-12-09T10:40:37,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46396 deadline: 1733740897822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:37,827 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:37,827 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:37,827 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:37,827 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:37,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46362 deadline: 1733740897824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:37,828 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:37,828 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:37,828 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:37,841 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:37,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46398 deadline: 1733740897830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:37,842 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:37,842 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:37,842 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:37,843 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:37,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46374 deadline: 1733740897830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:37,844 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:37,844 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:37,844 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:37,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741865_1041 (size=12151) 2024-12-09T10:40:37,847 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/B/2f73e1188d8b4a598b576b535fa20f34 2024-12-09T10:40:37,863 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/C/e4b4d95ea682449a9dbeeea3ff98dbf9 is 50, key is test_row_0/C:col10/1733740837492/Put/seqid=0 2024-12-09T10:40:37,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741866_1042 (size=12151) 2024-12-09T10:40:37,885 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/C/e4b4d95ea682449a9dbeeea3ff98dbf9 2024-12-09T10:40:37,914 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/A/84f13aafe378489c826b7afabf6b0dd4 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/84f13aafe378489c826b7afabf6b0dd4 2024-12-09T10:40:37,929 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/84f13aafe378489c826b7afabf6b0dd4, entries=200, sequenceid=170, filesize=14.2 K 2024-12-09T10:40:37,931 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/B/2f73e1188d8b4a598b576b535fa20f34 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/2f73e1188d8b4a598b576b535fa20f34 2024-12-09T10:40:37,935 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:37,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46396 deadline: 1733740897930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:37,936 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:37,936 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:37,936 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:37,936 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:37,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46362 deadline: 1733740897930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:37,937 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:37,937 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:37,937 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:37,943 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/2f73e1188d8b4a598b576b535fa20f34, entries=150, sequenceid=170, filesize=11.9 K 2024-12-09T10:40:37,945 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/C/e4b4d95ea682449a9dbeeea3ff98dbf9 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/e4b4d95ea682449a9dbeeea3ff98dbf9 2024-12-09T10:40:37,954 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/e4b4d95ea682449a9dbeeea3ff98dbf9, entries=150, sequenceid=170, filesize=11.9 K 2024-12-09T10:40:37,956 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~136.42 KB/139690, heapSize ~390.02 KB/399376, currentSize=46.96 KB/48090 for 1d9d6bba7dcd09663c7b08474fc1c1b5 in 191ms, sequenceid=170, compaction requested=true 2024-12-09T10:40:37,956 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1d9d6bba7dcd09663c7b08474fc1c1b5: 2024-12-09T10:40:37,956 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1d9d6bba7dcd09663c7b08474fc1c1b5:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T10:40:37,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:40:37,957 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T10:40:37,957 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T10:40:37,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1d9d6bba7dcd09663c7b08474fc1c1b5:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T10:40:37,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:40:37,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1d9d6bba7dcd09663c7b08474fc1c1b5:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T10:40:37,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T10:40:37,960 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48410 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T10:40:37,960 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53140 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T10:40:37,960 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1541): 1d9d6bba7dcd09663c7b08474fc1c1b5/B is initiating minor compaction (all files) 2024-12-09T10:40:37,960 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1541): 1d9d6bba7dcd09663c7b08474fc1c1b5/A is initiating minor compaction (all files) 2024-12-09T10:40:37,960 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1d9d6bba7dcd09663c7b08474fc1c1b5/B in TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:37,960 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1d9d6bba7dcd09663c7b08474fc1c1b5/A in TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:37,960 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/a31be429598e44d0a5b6399d21228c95, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/68dfe148d84d4c1a9f484a389dd84455, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/dc59d33a81a9401c807c211463f76a9a, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/2f73e1188d8b4a598b576b535fa20f34] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp, totalSize=47.3 K 2024-12-09T10:40:37,960 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/519baef860a141f2b3ec677210dad12d, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/d561b4afe4e14addb37708472b2ea07e, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/4ff24538ff4e428288740e499ae36eca, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/84f13aafe378489c826b7afabf6b0dd4] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp, totalSize=51.9 K 2024-12-09T10:40:37,961 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting a31be429598e44d0a5b6399d21228c95, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1733740834751 2024-12-09T10:40:37,961 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 519baef860a141f2b3ec677210dad12d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1733740834751 2024-12-09T10:40:37,962 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 68dfe148d84d4c1a9f484a389dd84455, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1733740835531 2024-12-09T10:40:37,962 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting d561b4afe4e14addb37708472b2ea07e, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1733740835531 2024-12-09T10:40:37,963 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting dc59d33a81a9401c807c211463f76a9a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1733740836215 2024-12-09T10:40:37,963 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ff24538ff4e428288740e499ae36eca, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1733740836215 2024-12-09T10:40:37,963 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f73e1188d8b4a598b576b535fa20f34, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1733740837491 2024-12-09T10:40:37,964 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 84f13aafe378489c826b7afabf6b0dd4, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1733740837480 2024-12-09T10:40:37,995 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1d9d6bba7dcd09663c7b08474fc1c1b5#A#compaction#30 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:40:37,995 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/A/a2f96124a504430b935eba66e24cca79 is 50, key is test_row_0/A:col10/1733740837492/Put/seqid=0 2024-12-09T10:40:38,004 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1d9d6bba7dcd09663c7b08474fc1c1b5#B#compaction#31 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:40:38,005 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/B/53cc132cb7e34283ab01482de04ae7da is 50, key is test_row_0/B:col10/1733740837492/Put/seqid=0 2024-12-09T10:40:38,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741867_1043 (size=12493) 2024-12-09T10:40:38,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741868_1044 (size=12493) 2024-12-09T10:40:38,042 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/B/53cc132cb7e34283ab01482de04ae7da as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/53cc132cb7e34283ab01482de04ae7da 2024-12-09T10:40:38,053 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/A/a2f96124a504430b935eba66e24cca79 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/a2f96124a504430b935eba66e24cca79 2024-12-09T10:40:38,055 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1d9d6bba7dcd09663c7b08474fc1c1b5/B of 1d9d6bba7dcd09663c7b08474fc1c1b5 into 53cc132cb7e34283ab01482de04ae7da(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:40:38,055 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1d9d6bba7dcd09663c7b08474fc1c1b5: 2024-12-09T10:40:38,055 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., storeName=1d9d6bba7dcd09663c7b08474fc1c1b5/B, priority=12, startTime=1733740837957; duration=0sec 2024-12-09T10:40:38,056 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T10:40:38,056 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1d9d6bba7dcd09663c7b08474fc1c1b5:B 2024-12-09T10:40:38,056 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T10:40:38,059 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48410 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T10:40:38,059 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1541): 1d9d6bba7dcd09663c7b08474fc1c1b5/C is initiating minor compaction (all files) 2024-12-09T10:40:38,060 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1d9d6bba7dcd09663c7b08474fc1c1b5/C in TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:38,060 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/6a86fc708dfb4b99926cd2728a25f1d5, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/ccd5118581644949a62cd9f2b852f010, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/307768823c2544879488b5dd67fe12c6, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/e4b4d95ea682449a9dbeeea3ff98dbf9] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp, totalSize=47.3 K 2024-12-09T10:40:38,062 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a86fc708dfb4b99926cd2728a25f1d5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1733740834751 2024-12-09T10:40:38,062 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting ccd5118581644949a62cd9f2b852f010, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1733740835531 2024-12-09T10:40:38,063 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 307768823c2544879488b5dd67fe12c6, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1733740836215 2024-12-09T10:40:38,064 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting e4b4d95ea682449a9dbeeea3ff98dbf9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1733740837491 2024-12-09T10:40:38,073 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1d9d6bba7dcd09663c7b08474fc1c1b5/A of 1d9d6bba7dcd09663c7b08474fc1c1b5 into a2f96124a504430b935eba66e24cca79(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:40:38,076 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1d9d6bba7dcd09663c7b08474fc1c1b5: 2024-12-09T10:40:38,076 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., storeName=1d9d6bba7dcd09663c7b08474fc1c1b5/A, priority=12, startTime=1733740837956; duration=0sec 2024-12-09T10:40:38,077 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:40:38,077 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1d9d6bba7dcd09663c7b08474fc1c1b5:A 2024-12-09T10:40:38,080 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1d9d6bba7dcd09663c7b08474fc1c1b5#C#compaction#32 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:40:38,080 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/C/c3b9b04788e64b6e88939f0af5d38970 is 50, key is test_row_0/C:col10/1733740837492/Put/seqid=0 2024-12-09T10:40:38,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741869_1045 (size=12493) 2024-12-09T10:40:38,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(8855): Flush requested on 1d9d6bba7dcd09663c7b08474fc1c1b5 2024-12-09T10:40:38,137 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1d9d6bba7dcd09663c7b08474fc1c1b5 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-09T10:40:38,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d9d6bba7dcd09663c7b08474fc1c1b5, store=A 2024-12-09T10:40:38,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:40:38,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d9d6bba7dcd09663c7b08474fc1c1b5, store=B 2024-12-09T10:40:38,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:40:38,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d9d6bba7dcd09663c7b08474fc1c1b5, store=C 2024-12-09T10:40:38,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:40:38,145 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/A/e52c0814ccf942fab4cf11bd76100872 is 50, key is test_row_0/A:col10/1733740837816/Put/seqid=0 2024-12-09T10:40:38,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741870_1046 (size=14541) 2024-12-09T10:40:38,192 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733740838177, max timestamp=1733740838186Num uniques 859; 2024-12-09T10:40:38,192 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T10:40:38,196 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733740838177, max timestamp=1733740838186Num uniques 859; 2024-12-09T10:40:38,196 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T10:40:38,200 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733740838177, max timestamp=1733740838186Num uniques 859; 2024-12-09T10:40:38,200 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T10:40:38,237 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:38,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46396 deadline: 1733740898230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:38,239 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:38,239 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:38,239 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:38,244 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:38,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46398 deadline: 1733740898233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:38,246 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:38,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46374 deadline: 1733740898235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:38,247 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:38,247 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:38,247 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:38,247 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:38,247 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:38,247 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:38,248 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:38,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46362 deadline: 1733740898238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:38,248 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:38,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46390 deadline: 1733740898238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:38,249 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:38,249 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:38,249 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:38,249 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:38,249 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:38,249 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:38,353 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:38,353 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:38,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46374 deadline: 1733740898350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:38,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46396 deadline: 1733740898350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:38,354 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:38,354 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:38,354 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:38,354 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:38,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46398 deadline: 1733740898350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:38,354 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:38,354 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:38,355 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:38,355 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:38,355 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:38,355 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:38,360 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:38,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46362 deadline: 1733740898360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:38,361 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:38,361 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:38,361 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:38,361 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:38,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46390 deadline: 1733740898360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:38,362 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:38,362 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:38,362 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:38,532 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/C/c3b9b04788e64b6e88939f0af5d38970 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/c3b9b04788e64b6e88939f0af5d38970 2024-12-09T10:40:38,542 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1d9d6bba7dcd09663c7b08474fc1c1b5/C of 1d9d6bba7dcd09663c7b08474fc1c1b5 into c3b9b04788e64b6e88939f0af5d38970(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:40:38,542 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1d9d6bba7dcd09663c7b08474fc1c1b5: 2024-12-09T10:40:38,542 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., storeName=1d9d6bba7dcd09663c7b08474fc1c1b5/C, priority=12, startTime=1733740837957; duration=0sec 2024-12-09T10:40:38,542 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:40:38,543 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1d9d6bba7dcd09663c7b08474fc1c1b5:C 2024-12-09T10:40:38,561 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:38,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46374 deadline: 1733740898560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:38,562 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:38,562 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:38,562 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:38,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46398 deadline: 1733740898560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:38,562 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:38,563 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:38,563 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:38,563 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:38,563 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:38,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46396 deadline: 1733740898560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:38,563 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:38,563 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:38,564 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:38,574 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:38,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46362 deadline: 1733740898570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:38,575 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:38,575 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:38,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46390 deadline: 1733740898570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:38,575 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:38,575 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:38,575 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:38,576 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:38,576 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:38,585 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/A/e52c0814ccf942fab4cf11bd76100872 2024-12-09T10:40:38,609 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/B/3c6ddfe19ae14e80987a3ff79e738f81 is 50, key is test_row_0/B:col10/1733740837816/Put/seqid=0 2024-12-09T10:40:38,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741871_1047 (size=12151) 2024-12-09T10:40:38,627 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/B/3c6ddfe19ae14e80987a3ff79e738f81 2024-12-09T10:40:38,643 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/C/f033ca332b6e4890b5aae6462dd5ddb5 is 50, key is test_row_0/C:col10/1733740837816/Put/seqid=0 2024-12-09T10:40:38,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741872_1048 (size=12151) 2024-12-09T10:40:38,659 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/C/f033ca332b6e4890b5aae6462dd5ddb5 2024-12-09T10:40:38,674 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/A/e52c0814ccf942fab4cf11bd76100872 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/e52c0814ccf942fab4cf11bd76100872 2024-12-09T10:40:38,684 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/e52c0814ccf942fab4cf11bd76100872, entries=200, sequenceid=183, filesize=14.2 K 2024-12-09T10:40:38,686 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/B/3c6ddfe19ae14e80987a3ff79e738f81 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/3c6ddfe19ae14e80987a3ff79e738f81 2024-12-09T10:40:38,695 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/3c6ddfe19ae14e80987a3ff79e738f81, entries=150, sequenceid=183, filesize=11.9 K 2024-12-09T10:40:38,697 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/C/f033ca332b6e4890b5aae6462dd5ddb5 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/f033ca332b6e4890b5aae6462dd5ddb5 2024-12-09T10:40:38,707 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/f033ca332b6e4890b5aae6462dd5ddb5, entries=150, sequenceid=183, filesize=11.9 K 2024-12-09T10:40:38,710 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 1d9d6bba7dcd09663c7b08474fc1c1b5 in 571ms, sequenceid=183, compaction requested=false 2024-12-09T10:40:38,710 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1d9d6bba7dcd09663c7b08474fc1c1b5: 2024-12-09T10:40:38,877 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733740838227, max timestamp=1733740838873Num uniques 1018; 2024-12-09T10:40:38,877 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-09T10:40:38,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(8855): Flush requested on 1d9d6bba7dcd09663c7b08474fc1c1b5 2024-12-09T10:40:38,878 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733740838227, max timestamp=1733740838873Num uniques 1018; 2024-12-09T10:40:38,878 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1d9d6bba7dcd09663c7b08474fc1c1b5 3/3 column families, dataSize=109.58 KB heapSize=312.13 KB 2024-12-09T10:40:38,878 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-09T10:40:38,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d9d6bba7dcd09663c7b08474fc1c1b5, store=A 2024-12-09T10:40:38,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-09T10:40:38,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d9d6bba7dcd09663c7b08474fc1c1b5, store=B 2024-12-09T10:40:38,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T10:40:38,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d9d6bba7dcd09663c7b08474fc1c1b5, store=C 2024-12-09T10:40:38,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T10:40:38,896 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/A/93a983b505074152996323bd7d6c8d00 is 50, key is test_row_0/A:col10/1733740838873/Put/seqid=0 2024-12-09T10:40:38,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741873_1049 (size=16931) 2024-12-09T10:40:38,905 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=223 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/A/93a983b505074152996323bd7d6c8d00 2024-12-09T10:40:38,924 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/B/067a265031794c249899bd83c2708289 is 50, key is test_row_0/B:col10/1733740838873/Put/seqid=0 2024-12-09T10:40:38,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741874_1050 (size=12151) 2024-12-09T10:40:38,981 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:38,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46396 deadline: 1733740898972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:38,982 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:38,982 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:38,982 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:38,982 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:38,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46362 deadline: 1733740898976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:38,983 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:38,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46398 deadline: 1733740898978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:38,983 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:38,983 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:38,983 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:38,984 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:38,984 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:38,984 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:38,985 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:38,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46374 deadline: 1733740898981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:38,986 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:38,986 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:38,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46390 deadline: 1733740898981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:38,987 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:38,987 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:38,987 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:38,986 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:38,989 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:39,092 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:39,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46396 deadline: 1733740899090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:39,093 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:39,093 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:39,093 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:39,093 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:39,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46362 deadline: 1733740899090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:39,093 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:39,094 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:39,094 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:39,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46398 deadline: 1733740899090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:39,094 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:39,094 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:39,094 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:39,094 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:39,099 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:39,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46390 deadline: 1733740899090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:39,099 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:39,099 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:39,100 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:39,103 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:39,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46374 deadline: 1733740899101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:39,104 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:39,104 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:39,104 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:39,301 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:39,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46396 deadline: 1733740899300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:39,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:39,302 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:39,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:39,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46362 deadline: 1733740899301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:39,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:39,303 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:39,303 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:39,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46398 deadline: 1733740899301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:39,303 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:39,303 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:39,304 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:39,304 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:39,304 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:39,313 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:39,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46390 deadline: 1733740899309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:39,314 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:39,314 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:39,314 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:39,314 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:39,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46374 deadline: 1733740899309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:39,314 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:39,314 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:39,314 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:39,334 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=223 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/B/067a265031794c249899bd83c2708289 2024-12-09T10:40:39,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-09T10:40:39,340 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T10:40:39,341 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-09T10:40:39,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-12-09T10:40:39,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-09T10:40:39,344 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T10:40:39,345 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T10:40:39,346 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T10:40:39,348 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/C/00d345660efd4f4f8f3e17405b072809 is 50, key is test_row_0/C:col10/1733740838873/Put/seqid=0 2024-12-09T10:40:39,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741875_1051 (size=12151) 2024-12-09T10:40:39,363 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=223 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/C/00d345660efd4f4f8f3e17405b072809 2024-12-09T10:40:39,375 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/A/93a983b505074152996323bd7d6c8d00 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/93a983b505074152996323bd7d6c8d00 2024-12-09T10:40:39,384 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/93a983b505074152996323bd7d6c8d00, entries=250, sequenceid=223, filesize=16.5 K 2024-12-09T10:40:39,387 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/B/067a265031794c249899bd83c2708289 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/067a265031794c249899bd83c2708289 2024-12-09T10:40:39,400 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/067a265031794c249899bd83c2708289, entries=150, sequenceid=223, filesize=11.9 K 2024-12-09T10:40:39,402 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/C/00d345660efd4f4f8f3e17405b072809 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/00d345660efd4f4f8f3e17405b072809 2024-12-09T10:40:39,411 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/00d345660efd4f4f8f3e17405b072809, entries=150, sequenceid=223, filesize=11.9 K 2024-12-09T10:40:39,413 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~71.56 KB/73280, heapSize ~198.67 KB/203440, currentSize=114.05 KB/116790 for 1d9d6bba7dcd09663c7b08474fc1c1b5 in 535ms, sequenceid=223, compaction requested=true 2024-12-09T10:40:39,413 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1d9d6bba7dcd09663c7b08474fc1c1b5: 2024-12-09T10:40:39,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1d9d6bba7dcd09663c7b08474fc1c1b5:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T10:40:39,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:40:39,414 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T10:40:39,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1d9d6bba7dcd09663c7b08474fc1c1b5:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T10:40:39,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:40:39,414 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T10:40:39,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1d9d6bba7dcd09663c7b08474fc1c1b5:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T10:40:39,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T10:40:39,416 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T10:40:39,416 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1541): 1d9d6bba7dcd09663c7b08474fc1c1b5/A is initiating minor compaction (all files) 2024-12-09T10:40:39,417 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1d9d6bba7dcd09663c7b08474fc1c1b5/A in TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:39,417 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/a2f96124a504430b935eba66e24cca79, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/e52c0814ccf942fab4cf11bd76100872, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/93a983b505074152996323bd7d6c8d00] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp, totalSize=42.9 K 2024-12-09T10:40:39,418 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T10:40:39,418 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1541): 1d9d6bba7dcd09663c7b08474fc1c1b5/B is initiating minor compaction (all files) 2024-12-09T10:40:39,418 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1d9d6bba7dcd09663c7b08474fc1c1b5/B in TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:39,418 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/53cc132cb7e34283ab01482de04ae7da, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/3c6ddfe19ae14e80987a3ff79e738f81, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/067a265031794c249899bd83c2708289] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp, totalSize=35.9 K 2024-12-09T10:40:39,419 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting a2f96124a504430b935eba66e24cca79, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1733740837491 2024-12-09T10:40:39,420 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 53cc132cb7e34283ab01482de04ae7da, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1733740837491 2024-12-09T10:40:39,421 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting e52c0814ccf942fab4cf11bd76100872, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1733740837799 2024-12-09T10:40:39,421 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c6ddfe19ae14e80987a3ff79e738f81, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1733740837799 2024-12-09T10:40:39,422 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 93a983b505074152996323bd7d6c8d00, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1733740838227 2024-12-09T10:40:39,423 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 067a265031794c249899bd83c2708289, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1733740838231 2024-12-09T10:40:39,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-09T10:40:39,451 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1d9d6bba7dcd09663c7b08474fc1c1b5#A#compaction#39 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:40:39,454 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1d9d6bba7dcd09663c7b08474fc1c1b5#B#compaction#40 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:40:39,454 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/A/aa60573fcd6343b7825ebc81e381af62 is 50, key is test_row_0/A:col10/1733740838873/Put/seqid=0 2024-12-09T10:40:39,454 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/B/ec70f635ad2b4a899042662b2e834420 is 50, key is test_row_0/B:col10/1733740838873/Put/seqid=0 2024-12-09T10:40:39,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741876_1052 (size=12595) 2024-12-09T10:40:39,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741877_1053 (size=12595) 2024-12-09T10:40:39,491 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/B/ec70f635ad2b4a899042662b2e834420 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/ec70f635ad2b4a899042662b2e834420 2024-12-09T10:40:39,499 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-09T10:40:39,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:39,500 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 1d9d6bba7dcd09663c7b08474fc1c1b5 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-09T10:40:39,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d9d6bba7dcd09663c7b08474fc1c1b5, store=A 2024-12-09T10:40:39,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:40:39,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d9d6bba7dcd09663c7b08474fc1c1b5, store=B 2024-12-09T10:40:39,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:40:39,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d9d6bba7dcd09663c7b08474fc1c1b5, store=C 2024-12-09T10:40:39,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:40:39,505 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1d9d6bba7dcd09663c7b08474fc1c1b5/B of 1d9d6bba7dcd09663c7b08474fc1c1b5 into ec70f635ad2b4a899042662b2e834420(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:40:39,506 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1d9d6bba7dcd09663c7b08474fc1c1b5: 2024-12-09T10:40:39,506 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., storeName=1d9d6bba7dcd09663c7b08474fc1c1b5/B, priority=13, startTime=1733740839414; duration=0sec 2024-12-09T10:40:39,506 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T10:40:39,506 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1d9d6bba7dcd09663c7b08474fc1c1b5:B 2024-12-09T10:40:39,506 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T10:40:39,508 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T10:40:39,508 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1541): 1d9d6bba7dcd09663c7b08474fc1c1b5/C is initiating minor compaction (all files) 2024-12-09T10:40:39,509 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1d9d6bba7dcd09663c7b08474fc1c1b5/C in TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:39,509 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/c3b9b04788e64b6e88939f0af5d38970, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/f033ca332b6e4890b5aae6462dd5ddb5, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/00d345660efd4f4f8f3e17405b072809] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp, totalSize=35.9 K 2024-12-09T10:40:39,509 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting c3b9b04788e64b6e88939f0af5d38970, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1733740837491 2024-12-09T10:40:39,510 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting f033ca332b6e4890b5aae6462dd5ddb5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1733740837799 2024-12-09T10:40:39,510 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 00d345660efd4f4f8f3e17405b072809, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1733740838231 2024-12-09T10:40:39,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/A/7ee5342ff436454b8b89e44cb0c96b65 is 50, key is test_row_0/A:col10/1733740838979/Put/seqid=0 2024-12-09T10:40:39,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741878_1054 (size=12151) 2024-12-09T10:40:39,541 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1d9d6bba7dcd09663c7b08474fc1c1b5#C#compaction#42 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:40:39,542 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/C/318a8bf8e663499c8bde325f4b071fed is 50, key is test_row_0/C:col10/1733740838873/Put/seqid=0 2024-12-09T10:40:39,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741879_1055 (size=12595) 2024-12-09T10:40:39,576 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/C/318a8bf8e663499c8bde325f4b071fed as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/318a8bf8e663499c8bde325f4b071fed 2024-12-09T10:40:39,588 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1d9d6bba7dcd09663c7b08474fc1c1b5/C of 1d9d6bba7dcd09663c7b08474fc1c1b5 into 318a8bf8e663499c8bde325f4b071fed(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:40:39,588 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1d9d6bba7dcd09663c7b08474fc1c1b5: 2024-12-09T10:40:39,589 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., storeName=1d9d6bba7dcd09663c7b08474fc1c1b5/C, priority=13, startTime=1733740839414; duration=0sec 2024-12-09T10:40:39,589 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:40:39,589 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1d9d6bba7dcd09663c7b08474fc1c1b5:C 2024-12-09T10:40:39,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(8855): Flush requested on 1d9d6bba7dcd09663c7b08474fc1c1b5 2024-12-09T10:40:39,614 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. as already flushing 2024-12-09T10:40:39,630 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:39,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46398 deadline: 1733740899625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:39,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:39,631 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:39,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46390 deadline: 1733740899627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:39,631 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:39,632 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:39,632 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:39,632 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:39,632 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:39,640 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:39,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46374 deadline: 1733740899631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:39,640 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:39,641 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:39,641 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:39,641 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:39,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46362 deadline: 1733740899631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:39,642 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:39,641 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:39,642 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:39,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46396 deadline: 1733740899632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:39,642 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:39,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:39,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:39,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:39,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-09T10:40:39,743 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:39,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46398 deadline: 1733740899740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:39,744 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:39,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:39,744 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:39,744 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:39,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46390 deadline: 1733740899740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:39,745 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:39,745 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:39,745 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:39,750 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:39,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46374 deadline: 1733740899749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:39,751 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:39,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46396 deadline: 1733740899750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:39,752 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:39,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:39,752 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:39,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:39,752 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:39,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:39,752 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:39,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46362 deadline: 1733740899750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:39,753 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:39,753 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:39,753 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:39,884 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/A/aa60573fcd6343b7825ebc81e381af62 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/aa60573fcd6343b7825ebc81e381af62 2024-12-09T10:40:39,891 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1d9d6bba7dcd09663c7b08474fc1c1b5/A of 1d9d6bba7dcd09663c7b08474fc1c1b5 into aa60573fcd6343b7825ebc81e381af62(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:40:39,891 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1d9d6bba7dcd09663c7b08474fc1c1b5: 2024-12-09T10:40:39,891 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., storeName=1d9d6bba7dcd09663c7b08474fc1c1b5/A, priority=13, startTime=1733740839414; duration=0sec 2024-12-09T10:40:39,892 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:40:39,892 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1d9d6bba7dcd09663c7b08474fc1c1b5:A 2024-12-09T10:40:39,942 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/A/7ee5342ff436454b8b89e44cb0c96b65 2024-12-09T10:40:39,951 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:39,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46398 deadline: 1733740899950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:39,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:39,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:39,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:39,952 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:39,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46390 deadline: 1733740899951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:39,953 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:39,953 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:39,953 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:39,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/B/da5ce49da32e4dd99699bba88114ece7 is 50, key is test_row_0/B:col10/1733740838979/Put/seqid=0 2024-12-09T10:40:39,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741880_1056 (size=12151) 2024-12-09T10:40:39,966 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:39,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46374 deadline: 1733740899959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:39,966 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:39,966 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:39,967 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:39,967 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:39,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46362 deadline: 1733740899960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:39,967 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:39,967 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:39,967 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:39,967 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:39,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46396 deadline: 1733740899960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:39,968 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:39,968 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:39,968 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:39,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-09T10:40:40,267 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:40,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46398 deadline: 1733740900260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:40,268 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:40,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46390 deadline: 1733740900260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:40,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:40,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:40,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:40,269 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:40,269 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:40,269 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:40,272 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:40,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46362 deadline: 1733740900270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:40,272 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:40,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46396 deadline: 1733740900270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:40,274 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:40,274 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:40,274 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:40,274 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:40,274 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:40,274 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:40,285 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:40,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46374 deadline: 1733740900279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:40,286 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:40,287 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:40,287 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:40,363 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/B/da5ce49da32e4dd99699bba88114ece7 2024-12-09T10:40:40,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/C/f7ab937338414073bdfd3ec3708b18af is 50, key is test_row_0/C:col10/1733740838979/Put/seqid=0 2024-12-09T10:40:40,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741881_1057 (size=12151) 2024-12-09T10:40:40,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-09T10:40:40,786 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:40,786 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:40,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46390 deadline: 1733740900780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:40,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46398 deadline: 1733740900780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:40,786 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:40,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46362 deadline: 1733740900780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:40,787 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:40,787 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:40,787 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:40,787 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:40,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46396 deadline: 1733740900781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:40,787 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:40,787 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:40,787 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:40,787 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:40,787 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:40,787 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:40,787 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:40,787 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:40,787 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:40,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:40,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46374 deadline: 1733740900789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:40,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:40,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:40,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:40,800 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/C/f7ab937338414073bdfd3ec3708b18af 2024-12-09T10:40:40,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/A/7ee5342ff436454b8b89e44cb0c96b65 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/7ee5342ff436454b8b89e44cb0c96b65 2024-12-09T10:40:40,822 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/7ee5342ff436454b8b89e44cb0c96b65, entries=150, sequenceid=244, filesize=11.9 K 2024-12-09T10:40:40,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/B/da5ce49da32e4dd99699bba88114ece7 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/da5ce49da32e4dd99699bba88114ece7 2024-12-09T10:40:40,841 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/da5ce49da32e4dd99699bba88114ece7, entries=150, sequenceid=244, filesize=11.9 K 2024-12-09T10:40:40,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/C/f7ab937338414073bdfd3ec3708b18af as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/f7ab937338414073bdfd3ec3708b18af 2024-12-09T10:40:40,849 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/f7ab937338414073bdfd3ec3708b18af, entries=150, sequenceid=244, filesize=11.9 K 2024-12-09T10:40:40,852 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for 1d9d6bba7dcd09663c7b08474fc1c1b5 in 1352ms, sequenceid=244, compaction requested=false 2024-12-09T10:40:40,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 1d9d6bba7dcd09663c7b08474fc1c1b5: 2024-12-09T10:40:40,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:40,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-12-09T10:40:40,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-12-09T10:40:40,858 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-12-09T10:40:40,858 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5090 sec 2024-12-09T10:40:40,861 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 1.5180 sec 2024-12-09T10:40:41,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-09T10:40:41,490 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T10:40:41,490 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-09T10:40:41,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-12-09T10:40:41,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-09T10:40:41,494 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T10:40:41,495 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T10:40:41,495 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T10:40:41,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-09T10:40:41,648 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-09T10:40:41,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:41,649 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 1d9d6bba7dcd09663c7b08474fc1c1b5 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-09T10:40:41,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d9d6bba7dcd09663c7b08474fc1c1b5, store=A 2024-12-09T10:40:41,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:40:41,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d9d6bba7dcd09663c7b08474fc1c1b5, store=B 2024-12-09T10:40:41,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:40:41,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d9d6bba7dcd09663c7b08474fc1c1b5, store=C 2024-12-09T10:40:41,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:40:41,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/A/3e90eb72c0fa44d595da746144634104 is 50, key is test_row_0/A:col10/1733740839621/Put/seqid=0 2024-12-09T10:40:41,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741882_1058 (size=12201) 2024-12-09T10:40:41,665 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/A/3e90eb72c0fa44d595da746144634104 2024-12-09T10:40:41,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/B/a8e0dbea74d64b74aa2ff244839c6072 is 50, key is test_row_0/B:col10/1733740839621/Put/seqid=0 2024-12-09T10:40:41,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741883_1059 (size=12201) 2024-12-09T10:40:41,693 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/B/a8e0dbea74d64b74aa2ff244839c6072 2024-12-09T10:40:41,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/C/753d37e30ca542e28deefc29494bc204 is 50, key is test_row_0/C:col10/1733740839621/Put/seqid=0 2024-12-09T10:40:41,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741884_1060 (size=12201) 2024-12-09T10:40:41,714 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/C/753d37e30ca542e28deefc29494bc204 2024-12-09T10:40:41,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/A/3e90eb72c0fa44d595da746144634104 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/3e90eb72c0fa44d595da746144634104 2024-12-09T10:40:41,729 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/3e90eb72c0fa44d595da746144634104, entries=150, sequenceid=259, filesize=11.9 K 2024-12-09T10:40:41,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/B/a8e0dbea74d64b74aa2ff244839c6072 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/a8e0dbea74d64b74aa2ff244839c6072 2024-12-09T10:40:41,737 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/a8e0dbea74d64b74aa2ff244839c6072, entries=150, sequenceid=259, filesize=11.9 K 2024-12-09T10:40:41,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/C/753d37e30ca542e28deefc29494bc204 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/753d37e30ca542e28deefc29494bc204 2024-12-09T10:40:41,746 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/753d37e30ca542e28deefc29494bc204, entries=150, sequenceid=259, filesize=11.9 K 2024-12-09T10:40:41,747 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=0 B/0 for 1d9d6bba7dcd09663c7b08474fc1c1b5 in 99ms, sequenceid=259, compaction requested=true 2024-12-09T10:40:41,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 1d9d6bba7dcd09663c7b08474fc1c1b5: 2024-12-09T10:40:41,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:41,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-12-09T10:40:41,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-12-09T10:40:41,753 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-12-09T10:40:41,753 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 255 msec 2024-12-09T10:40:41,759 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 266 msec 2024-12-09T10:40:41,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-09T10:40:41,814 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T10:40:41,815 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-09T10:40:41,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-12-09T10:40:41,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(8855): Flush requested on 1d9d6bba7dcd09663c7b08474fc1c1b5 2024-12-09T10:40:41,817 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1d9d6bba7dcd09663c7b08474fc1c1b5 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-09T10:40:41,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-09T10:40:41,818 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T10:40:41,819 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d9d6bba7dcd09663c7b08474fc1c1b5, store=A 2024-12-09T10:40:41,819 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:40:41,819 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d9d6bba7dcd09663c7b08474fc1c1b5, store=B 2024-12-09T10:40:41,819 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:40:41,819 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T10:40:41,819 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d9d6bba7dcd09663c7b08474fc1c1b5, store=C 2024-12-09T10:40:41,819 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:40:41,819 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T10:40:41,830 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/A/f2f6c41707c74aaaaa98e6f7a67c738e is 50, key is test_row_0/A:col10/1733740841816/Put/seqid=0 2024-12-09T10:40:41,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741885_1061 (size=17181) 2024-12-09T10:40:41,840 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/A/f2f6c41707c74aaaaa98e6f7a67c738e 2024-12-09T10:40:41,854 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:41,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46390 deadline: 1733740901848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:41,856 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/B/ecc129019e3e46d49766d4ca5d05c005 is 50, key is test_row_0/B:col10/1733740841816/Put/seqid=0 2024-12-09T10:40:41,856 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733740841843, max timestamp=1733740841852Num uniques 859; 2024-12-09T10:40:41,856 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:41,856 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-09T10:40:41,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46362 deadline: 1733740901850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:41,857 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733740841843, max timestamp=1733740841852Num uniques 859; 2024-12-09T10:40:41,857 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:41,857 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:41,857 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-09T10:40:41,857 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:41,858 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:41,858 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:41,858 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:41,859 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733740841843, max timestamp=1733740841852Num uniques 859; 2024-12-09T10:40:41,860 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-09T10:40:41,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741886_1062 (size=12301) 2024-12-09T10:40:41,865 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/B/ecc129019e3e46d49766d4ca5d05c005 2024-12-09T10:40:41,878 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/C/315a02c3b36e435e8c38c8a7ac910e9d is 50, key is test_row_0/C:col10/1733740841816/Put/seqid=0 2024-12-09T10:40:41,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741887_1063 (size=12301) 2024-12-09T10:40:41,884 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/C/315a02c3b36e435e8c38c8a7ac910e9d 2024-12-09T10:40:41,892 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/A/f2f6c41707c74aaaaa98e6f7a67c738e as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/f2f6c41707c74aaaaa98e6f7a67c738e 2024-12-09T10:40:41,903 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/f2f6c41707c74aaaaa98e6f7a67c738e, entries=250, sequenceid=270, filesize=16.8 K 2024-12-09T10:40:41,907 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/B/ecc129019e3e46d49766d4ca5d05c005 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/ecc129019e3e46d49766d4ca5d05c005 2024-12-09T10:40:41,914 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/ecc129019e3e46d49766d4ca5d05c005, entries=150, sequenceid=270, filesize=12.0 K 2024-12-09T10:40:41,916 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/C/315a02c3b36e435e8c38c8a7ac910e9d as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/315a02c3b36e435e8c38c8a7ac910e9d 2024-12-09T10:40:41,916 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:41,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46398 deadline: 1733740901904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:41,918 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:41,918 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:41,918 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:41,918 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:41,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46374 deadline: 1733740901905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:41,919 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:41,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46396 deadline: 1733740901906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:41,919 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:41,919 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:41,919 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:41,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-09T10:40:41,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:41,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:41,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:41,934 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/315a02c3b36e435e8c38c8a7ac910e9d, entries=150, sequenceid=270, filesize=12.0 K 2024-12-09T10:40:41,936 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 1d9d6bba7dcd09663c7b08474fc1c1b5 in 120ms, sequenceid=270, compaction requested=true 2024-12-09T10:40:41,937 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1d9d6bba7dcd09663c7b08474fc1c1b5: 2024-12-09T10:40:41,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1d9d6bba7dcd09663c7b08474fc1c1b5:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T10:40:41,937 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T10:40:41,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:40:41,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1d9d6bba7dcd09663c7b08474fc1c1b5:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T10:40:41,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:40:41,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1d9d6bba7dcd09663c7b08474fc1c1b5:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T10:40:41,937 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T10:40:41,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T10:40:41,940 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54128 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T10:40:41,940 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1541): 1d9d6bba7dcd09663c7b08474fc1c1b5/A is initiating minor compaction (all files) 2024-12-09T10:40:41,940 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1d9d6bba7dcd09663c7b08474fc1c1b5/A in TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:41,940 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/aa60573fcd6343b7825ebc81e381af62, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/7ee5342ff436454b8b89e44cb0c96b65, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/3e90eb72c0fa44d595da746144634104, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/f2f6c41707c74aaaaa98e6f7a67c738e] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp, totalSize=52.9 K 2024-12-09T10:40:41,941 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49248 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T10:40:41,941 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa60573fcd6343b7825ebc81e381af62, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1733740838231 2024-12-09T10:40:41,941 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1541): 1d9d6bba7dcd09663c7b08474fc1c1b5/B is initiating minor compaction (all files) 2024-12-09T10:40:41,941 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1d9d6bba7dcd09663c7b08474fc1c1b5/B in TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:41,941 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/ec70f635ad2b4a899042662b2e834420, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/da5ce49da32e4dd99699bba88114ece7, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/a8e0dbea74d64b74aa2ff244839c6072, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/ecc129019e3e46d49766d4ca5d05c005] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp, totalSize=48.1 K 2024-12-09T10:40:41,941 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7ee5342ff436454b8b89e44cb0c96b65, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1733740838956 2024-12-09T10:40:41,942 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting ec70f635ad2b4a899042662b2e834420, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1733740838231 2024-12-09T10:40:41,942 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3e90eb72c0fa44d595da746144634104, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1733740839621 2024-12-09T10:40:41,942 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting da5ce49da32e4dd99699bba88114ece7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1733740838956 2024-12-09T10:40:41,942 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting f2f6c41707c74aaaaa98e6f7a67c738e, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1733740841802 2024-12-09T10:40:41,943 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting a8e0dbea74d64b74aa2ff244839c6072, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1733740839621 2024-12-09T10:40:41,944 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting ecc129019e3e46d49766d4ca5d05c005, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1733740841807 2024-12-09T10:40:41,965 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1d9d6bba7dcd09663c7b08474fc1c1b5#B#compaction#51 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:40:41,965 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1d9d6bba7dcd09663c7b08474fc1c1b5 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-12-09T10:40:41,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d9d6bba7dcd09663c7b08474fc1c1b5, store=A 2024-12-09T10:40:41,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T10:40:41,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d9d6bba7dcd09663c7b08474fc1c1b5, store=B 2024-12-09T10:40:41,966 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/B/e80496f306ee416cb1605ffa8f3bcc60 is 50, key is test_row_0/B:col10/1733740841816/Put/seqid=0 2024-12-09T10:40:41,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T10:40:41,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d9d6bba7dcd09663c7b08474fc1c1b5, store=C 2024-12-09T10:40:41,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T10:40:41,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(8855): Flush requested on 1d9d6bba7dcd09663c7b08474fc1c1b5 2024-12-09T10:40:41,972 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1d9d6bba7dcd09663c7b08474fc1c1b5#A#compaction#52 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:40:41,972 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/A/59aa8905b394496cbf17b87fbabb424b is 50, key is test_row_0/A:col10/1733740841816/Put/seqid=0 2024-12-09T10:40:41,977 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-09T10:40:41,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:41,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. as already flushing 2024-12-09T10:40:41,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:41,978 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:40:41,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:40:41,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:40:41,980 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/A/72ad68ed977f45a7825c746ac4813094 is 50, key is test_row_0/A:col10/1733740841889/Put/seqid=0 2024-12-09T10:40:41,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741888_1064 (size=12881) 2024-12-09T10:40:42,008 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:42,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46390 deadline: 1733740902006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:42,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:42,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46362 deadline: 1733740902008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:42,009 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:42,009 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:42,010 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:42,010 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:42,010 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:42,011 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:42,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741889_1065 (size=12881) 2024-12-09T10:40:42,032 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:42,032 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/A/59aa8905b394496cbf17b87fbabb424b as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/59aa8905b394496cbf17b87fbabb424b 2024-12-09T10:40:42,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46398 deadline: 1733740902030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:42,033 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:42,033 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:42,033 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:42,033 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:42,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46374 deadline: 1733740902030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:42,033 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:42,033 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:42,033 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:42,034 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:42,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46396 deadline: 1733740902031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:42,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:42,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:42,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:42,042 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1d9d6bba7dcd09663c7b08474fc1c1b5/A of 1d9d6bba7dcd09663c7b08474fc1c1b5 into 59aa8905b394496cbf17b87fbabb424b(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:40:42,042 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1d9d6bba7dcd09663c7b08474fc1c1b5: 2024-12-09T10:40:42,042 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., storeName=1d9d6bba7dcd09663c7b08474fc1c1b5/A, priority=12, startTime=1733740841937; duration=0sec 2024-12-09T10:40:42,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741890_1066 (size=14741) 2024-12-09T10:40:42,043 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T10:40:42,043 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1d9d6bba7dcd09663c7b08474fc1c1b5:A 2024-12-09T10:40:42,043 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T10:40:42,045 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=305 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/A/72ad68ed977f45a7825c746ac4813094 2024-12-09T10:40:42,046 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49248 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T10:40:42,046 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1541): 1d9d6bba7dcd09663c7b08474fc1c1b5/C is initiating minor compaction (all files) 2024-12-09T10:40:42,046 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1d9d6bba7dcd09663c7b08474fc1c1b5/C in TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:42,046 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/318a8bf8e663499c8bde325f4b071fed, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/f7ab937338414073bdfd3ec3708b18af, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/753d37e30ca542e28deefc29494bc204, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/315a02c3b36e435e8c38c8a7ac910e9d] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp, totalSize=48.1 K 2024-12-09T10:40:42,050 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 318a8bf8e663499c8bde325f4b071fed, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1733740838231 2024-12-09T10:40:42,051 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting f7ab937338414073bdfd3ec3708b18af, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1733740838956 2024-12-09T10:40:42,052 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 753d37e30ca542e28deefc29494bc204, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1733740839621 2024-12-09T10:40:42,054 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 315a02c3b36e435e8c38c8a7ac910e9d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1733740841807 2024-12-09T10:40:42,061 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/B/a800d2a76d15439799e5df23170fe085 is 50, key is test_row_0/B:col10/1733740841889/Put/seqid=0 2024-12-09T10:40:42,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741891_1067 (size=12301) 2024-12-09T10:40:42,087 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=305 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/B/a800d2a76d15439799e5df23170fe085 2024-12-09T10:40:42,093 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1d9d6bba7dcd09663c7b08474fc1c1b5#C#compaction#55 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:40:42,093 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/C/b90152d88b684ec9944339d615ad6d39 is 50, key is test_row_0/C:col10/1733740841816/Put/seqid=0 2024-12-09T10:40:42,101 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/C/614f39bffa314dd9a4d398542db4245c is 50, key is test_row_0/C:col10/1733740841889/Put/seqid=0 2024-12-09T10:40:42,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741892_1068 (size=12881) 2024-12-09T10:40:42,121 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:42,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741893_1069 (size=12301) 2024-12-09T10:40:42,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46390 deadline: 1733740902120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:42,122 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:42,122 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:42,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46362 deadline: 1733740902120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:42,122 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:42,122 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:42,122 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=305 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/C/614f39bffa314dd9a4d398542db4245c 2024-12-09T10:40:42,122 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:42,122 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:42,123 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:42,125 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/C/b90152d88b684ec9944339d615ad6d39 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/b90152d88b684ec9944339d615ad6d39 2024-12-09T10:40:42,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-09T10:40:42,133 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-09T10:40:42,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:42,134 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/A/72ad68ed977f45a7825c746ac4813094 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/72ad68ed977f45a7825c746ac4813094 2024-12-09T10:40:42,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. as already flushing 2024-12-09T10:40:42,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:42,134 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:40:42,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:40:42,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:40:42,139 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1d9d6bba7dcd09663c7b08474fc1c1b5/C of 1d9d6bba7dcd09663c7b08474fc1c1b5 into b90152d88b684ec9944339d615ad6d39(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:40:42,140 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1d9d6bba7dcd09663c7b08474fc1c1b5: 2024-12-09T10:40:42,140 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., storeName=1d9d6bba7dcd09663c7b08474fc1c1b5/C, priority=12, startTime=1733740841937; duration=0sec 2024-12-09T10:40:42,140 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:40:42,140 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1d9d6bba7dcd09663c7b08474fc1c1b5:C 2024-12-09T10:40:42,146 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/72ad68ed977f45a7825c746ac4813094, entries=200, sequenceid=305, filesize=14.4 K 2024-12-09T10:40:42,147 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/B/a800d2a76d15439799e5df23170fe085 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/a800d2a76d15439799e5df23170fe085 2024-12-09T10:40:42,159 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/a800d2a76d15439799e5df23170fe085, entries=150, sequenceid=305, filesize=12.0 K 2024-12-09T10:40:42,161 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/C/614f39bffa314dd9a4d398542db4245c as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/614f39bffa314dd9a4d398542db4245c 2024-12-09T10:40:42,170 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/614f39bffa314dd9a4d398542db4245c, entries=150, sequenceid=305, filesize=12.0 K 2024-12-09T10:40:42,172 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=53.67 KB/54960 for 1d9d6bba7dcd09663c7b08474fc1c1b5 in 206ms, sequenceid=305, compaction requested=false 2024-12-09T10:40:42,172 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1d9d6bba7dcd09663c7b08474fc1c1b5: 2024-12-09T10:40:42,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(8855): Flush requested on 1d9d6bba7dcd09663c7b08474fc1c1b5 2024-12-09T10:40:42,242 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1d9d6bba7dcd09663c7b08474fc1c1b5 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-09T10:40:42,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d9d6bba7dcd09663c7b08474fc1c1b5, store=A 2024-12-09T10:40:42,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:40:42,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d9d6bba7dcd09663c7b08474fc1c1b5, store=B 2024-12-09T10:40:42,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:40:42,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d9d6bba7dcd09663c7b08474fc1c1b5, store=C 2024-12-09T10:40:42,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:40:42,254 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/A/d4720f52842c40109685b8bd4344b28b is 50, key is test_row_1/A:col10/1733740842241/Put/seqid=0 2024-12-09T10:40:42,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741894_1070 (size=14737) 2024-12-09T10:40:42,270 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/A/d4720f52842c40109685b8bd4344b28b 2024-12-09T10:40:42,288 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/B/64abba655f484942991c9d17ceda5070 is 50, key is test_row_1/B:col10/1733740842241/Put/seqid=0 2024-12-09T10:40:42,289 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-09T10:40:42,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:42,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. as already flushing 2024-12-09T10:40:42,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:42,290 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:40:42,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:40:42,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:40:42,295 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:42,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46398 deadline: 1733740902290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:42,296 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:42,296 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:42,296 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:42,301 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:42,302 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:42,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46396 deadline: 1733740902296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:42,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46374 deadline: 1733740902296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:42,303 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:42,303 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:42,303 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:42,304 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:42,304 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:42,304 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:42,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741895_1071 (size=9857) 2024-12-09T10:40:42,309 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/B/64abba655f484942991c9d17ceda5070 2024-12-09T10:40:42,322 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/C/2db4c5508f0a4846adcecdef1a641470 is 50, key is test_row_1/C:col10/1733740842241/Put/seqid=0 2024-12-09T10:40:42,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741896_1072 (size=9857) 2024-12-09T10:40:42,331 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:42,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46390 deadline: 1733740902330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:42,334 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:42,334 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:42,334 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:42,334 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:42,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46362 deadline: 1733740902330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:42,336 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:42,336 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:42,336 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:42,402 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:42,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46398 deadline: 1733740902400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:42,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:42,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:42,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:42,405 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/B/e80496f306ee416cb1605ffa8f3bcc60 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/e80496f306ee416cb1605ffa8f3bcc60 2024-12-09T10:40:42,411 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:42,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46374 deadline: 1733740902410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:42,412 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:42,412 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:42,412 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:42,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46396 deadline: 1733740902410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:42,412 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1d9d6bba7dcd09663c7b08474fc1c1b5/B of 1d9d6bba7dcd09663c7b08474fc1c1b5 into e80496f306ee416cb1605ffa8f3bcc60(size=12.6 K), total size for store is 24.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:40:42,412 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:42,412 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1d9d6bba7dcd09663c7b08474fc1c1b5: 2024-12-09T10:40:42,412 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., storeName=1d9d6bba7dcd09663c7b08474fc1c1b5/B, priority=12, startTime=1733740841937; duration=0sec 2024-12-09T10:40:42,413 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:42,413 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:42,413 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:42,414 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:40:42,414 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1d9d6bba7dcd09663c7b08474fc1c1b5:B 2024-12-09T10:40:42,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-09T10:40:42,444 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-09T10:40:42,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:42,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. as already flushing 2024-12-09T10:40:42,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:42,445 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:40:42,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:40:42,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:40:42,599 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-09T10:40:42,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:42,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. as already flushing 2024-12-09T10:40:42,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:42,599 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:40:42,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:40:42,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:40:42,615 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:42,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46398 deadline: 1733740902610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:42,615 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:42,616 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:42,616 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:42,620 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:42,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46374 deadline: 1733740902620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:42,621 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:42,621 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:42,621 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:42,621 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:42,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46396 deadline: 1733740902620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:42,622 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:42,622 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:42,622 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:42,641 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:42,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46390 deadline: 1733740902640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:42,642 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:42,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46362 deadline: 1733740902640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:42,642 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:42,642 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:42,642 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:42,642 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:42,643 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:42,643 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:42,733 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/C/2db4c5508f0a4846adcecdef1a641470 2024-12-09T10:40:42,740 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/A/d4720f52842c40109685b8bd4344b28b as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/d4720f52842c40109685b8bd4344b28b 2024-12-09T10:40:42,750 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/d4720f52842c40109685b8bd4344b28b, entries=200, sequenceid=321, filesize=14.4 K 2024-12-09T10:40:42,752 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/B/64abba655f484942991c9d17ceda5070 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/64abba655f484942991c9d17ceda5070 2024-12-09T10:40:42,755 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-09T10:40:42,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:42,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. as already flushing 2024-12-09T10:40:42,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:42,755 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:40:42,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:40:42,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:40:42,763 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/64abba655f484942991c9d17ceda5070, entries=100, sequenceid=321, filesize=9.6 K 2024-12-09T10:40:42,764 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/C/2db4c5508f0a4846adcecdef1a641470 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/2db4c5508f0a4846adcecdef1a641470 2024-12-09T10:40:42,771 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/2db4c5508f0a4846adcecdef1a641470, entries=100, sequenceid=321, filesize=9.6 K 2024-12-09T10:40:42,773 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for 1d9d6bba7dcd09663c7b08474fc1c1b5 in 530ms, sequenceid=321, compaction requested=true 2024-12-09T10:40:42,773 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1d9d6bba7dcd09663c7b08474fc1c1b5: 2024-12-09T10:40:42,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1d9d6bba7dcd09663c7b08474fc1c1b5:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T10:40:42,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:40:42,773 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T10:40:42,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1d9d6bba7dcd09663c7b08474fc1c1b5:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T10:40:42,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:40:42,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1d9d6bba7dcd09663c7b08474fc1c1b5:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T10:40:42,773 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T10:40:42,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T10:40:42,775 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T10:40:42,775 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35039 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T10:40:42,775 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1541): 1d9d6bba7dcd09663c7b08474fc1c1b5/A is initiating minor compaction (all files) 2024-12-09T10:40:42,775 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1541): 1d9d6bba7dcd09663c7b08474fc1c1b5/B is initiating minor compaction (all files) 2024-12-09T10:40:42,775 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1d9d6bba7dcd09663c7b08474fc1c1b5/A in TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:42,775 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1d9d6bba7dcd09663c7b08474fc1c1b5/B in TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:42,775 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/59aa8905b394496cbf17b87fbabb424b, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/72ad68ed977f45a7825c746ac4813094, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/d4720f52842c40109685b8bd4344b28b] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp, totalSize=41.4 K 2024-12-09T10:40:42,775 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/e80496f306ee416cb1605ffa8f3bcc60, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/a800d2a76d15439799e5df23170fe085, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/64abba655f484942991c9d17ceda5070] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp, totalSize=34.2 K 2024-12-09T10:40:42,776 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 59aa8905b394496cbf17b87fbabb424b, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1733740841807 2024-12-09T10:40:42,776 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting e80496f306ee416cb1605ffa8f3bcc60, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1733740841807 2024-12-09T10:40:42,776 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 72ad68ed977f45a7825c746ac4813094, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1733740841889 2024-12-09T10:40:42,776 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting a800d2a76d15439799e5df23170fe085, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1733740841889 2024-12-09T10:40:42,777 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting d4720f52842c40109685b8bd4344b28b, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1733740842002 2024-12-09T10:40:42,777 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 64abba655f484942991c9d17ceda5070, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1733740842240 2024-12-09T10:40:42,788 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1d9d6bba7dcd09663c7b08474fc1c1b5#A#compaction#60 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:40:42,789 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/A/4f52605b6b144183b8544119e0e14e12 is 50, key is test_row_0/A:col10/1733740841889/Put/seqid=0 2024-12-09T10:40:42,792 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1d9d6bba7dcd09663c7b08474fc1c1b5#B#compaction#61 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:40:42,793 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/B/b3656d6396a94ffdb3e4ce3b0750b9b3 is 50, key is test_row_0/B:col10/1733740841889/Put/seqid=0 2024-12-09T10:40:42,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741897_1073 (size=12983) 2024-12-09T10:40:42,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741898_1074 (size=12983) 2024-12-09T10:40:42,909 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-09T10:40:42,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:42,910 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 1d9d6bba7dcd09663c7b08474fc1c1b5 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-09T10:40:42,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d9d6bba7dcd09663c7b08474fc1c1b5, store=A 2024-12-09T10:40:42,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:40:42,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d9d6bba7dcd09663c7b08474fc1c1b5, store=B 2024-12-09T10:40:42,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:40:42,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d9d6bba7dcd09663c7b08474fc1c1b5, store=C 2024-12-09T10:40:42,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:40:42,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/A/0135ce2f1414452eaa260c2232a8d4d7 is 50, key is test_row_0/A:col10/1733740842286/Put/seqid=0 2024-12-09T10:40:42,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(8855): Flush requested on 1d9d6bba7dcd09663c7b08474fc1c1b5 2024-12-09T10:40:42,926 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. as already flushing 2024-12-09T10:40:42,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741899_1075 (size=12301) 2024-12-09T10:40:42,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-09T10:40:42,952 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:42,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46398 deadline: 1733740902946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:42,953 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:42,953 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:42,953 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:42,958 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:42,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46374 deadline: 1733740902950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:42,959 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:42,959 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:42,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:42,959 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:42,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46396 deadline: 1733740902953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:42,960 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:42,960 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:42,960 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:43,060 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:43,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46398 deadline: 1733740903060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:43,061 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:43,061 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:43,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:43,072 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:43,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46374 deadline: 1733740903069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:43,073 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:43,073 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:43,073 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:43,073 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:43,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46396 deadline: 1733740903070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:43,073 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:43,074 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:43,074 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:43,150 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:43,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46390 deadline: 1733740903150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:43,151 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:43,151 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:43,151 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:43,155 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:43,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46362 deadline: 1733740903150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:43,155 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:43,155 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:43,155 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:43,206 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/A/4f52605b6b144183b8544119e0e14e12 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/4f52605b6b144183b8544119e0e14e12 2024-12-09T10:40:43,213 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/B/b3656d6396a94ffdb3e4ce3b0750b9b3 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/b3656d6396a94ffdb3e4ce3b0750b9b3 2024-12-09T10:40:43,213 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1d9d6bba7dcd09663c7b08474fc1c1b5/A of 1d9d6bba7dcd09663c7b08474fc1c1b5 into 4f52605b6b144183b8544119e0e14e12(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:40:43,214 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1d9d6bba7dcd09663c7b08474fc1c1b5: 2024-12-09T10:40:43,214 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., storeName=1d9d6bba7dcd09663c7b08474fc1c1b5/A, priority=13, startTime=1733740842773; duration=0sec 2024-12-09T10:40:43,214 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T10:40:43,214 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1d9d6bba7dcd09663c7b08474fc1c1b5:A 2024-12-09T10:40:43,214 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T10:40:43,217 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35039 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T10:40:43,217 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1541): 1d9d6bba7dcd09663c7b08474fc1c1b5/C is initiating minor compaction (all files) 2024-12-09T10:40:43,217 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1d9d6bba7dcd09663c7b08474fc1c1b5/C in TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:43,217 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/b90152d88b684ec9944339d615ad6d39, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/614f39bffa314dd9a4d398542db4245c, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/2db4c5508f0a4846adcecdef1a641470] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp, totalSize=34.2 K 2024-12-09T10:40:43,218 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting b90152d88b684ec9944339d615ad6d39, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1733740841807 2024-12-09T10:40:43,219 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 614f39bffa314dd9a4d398542db4245c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1733740841889 2024-12-09T10:40:43,219 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2db4c5508f0a4846adcecdef1a641470, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1733740842240 2024-12-09T10:40:43,221 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1d9d6bba7dcd09663c7b08474fc1c1b5/B of 1d9d6bba7dcd09663c7b08474fc1c1b5 into b3656d6396a94ffdb3e4ce3b0750b9b3(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:40:43,221 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1d9d6bba7dcd09663c7b08474fc1c1b5: 2024-12-09T10:40:43,221 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., storeName=1d9d6bba7dcd09663c7b08474fc1c1b5/B, priority=13, startTime=1733740842773; duration=0sec 2024-12-09T10:40:43,221 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:40:43,221 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1d9d6bba7dcd09663c7b08474fc1c1b5:B 2024-12-09T10:40:43,229 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1d9d6bba7dcd09663c7b08474fc1c1b5#C#compaction#63 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:40:43,229 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/C/75e17d09da42431cb6836abb3a9a2323 is 50, key is test_row_0/C:col10/1733740841889/Put/seqid=0 2024-12-09T10:40:43,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741900_1076 (size=12983) 2024-12-09T10:40:43,271 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:43,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46398 deadline: 1733740903270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:43,271 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:43,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:43,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:43,283 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:43,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46374 deadline: 1733740903279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:43,284 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:43,284 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:43,284 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:43,284 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:43,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46396 deadline: 1733740903279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:43,284 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:43,284 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:43,284 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:43,334 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=341 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/A/0135ce2f1414452eaa260c2232a8d4d7 2024-12-09T10:40:43,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/B/e30a28c65b274a589bb52c39d979de47 is 50, key is test_row_0/B:col10/1733740842286/Put/seqid=0 2024-12-09T10:40:43,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741901_1077 (size=12301) 2024-12-09T10:40:43,585 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:43,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46398 deadline: 1733740903580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:43,586 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:43,586 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:43,586 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:43,590 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:43,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46374 deadline: 1733740903589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:43,590 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:43,590 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:43,590 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:43,594 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:43,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46396 deadline: 1733740903589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:43,595 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:43,595 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:43,595 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:43,641 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/C/75e17d09da42431cb6836abb3a9a2323 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/75e17d09da42431cb6836abb3a9a2323 2024-12-09T10:40:43,649 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1d9d6bba7dcd09663c7b08474fc1c1b5/C of 1d9d6bba7dcd09663c7b08474fc1c1b5 into 75e17d09da42431cb6836abb3a9a2323(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:40:43,649 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1d9d6bba7dcd09663c7b08474fc1c1b5: 2024-12-09T10:40:43,649 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., storeName=1d9d6bba7dcd09663c7b08474fc1c1b5/C, priority=13, startTime=1733740842773; duration=0sec 2024-12-09T10:40:43,650 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:40:43,650 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1d9d6bba7dcd09663c7b08474fc1c1b5:C 2024-12-09T10:40:43,750 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=341 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/B/e30a28c65b274a589bb52c39d979de47 2024-12-09T10:40:43,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/C/08070b46c9c94c2e978a3ade9226f7a1 is 50, key is test_row_0/C:col10/1733740842286/Put/seqid=0 2024-12-09T10:40:43,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741902_1078 (size=12301) 2024-12-09T10:40:43,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-09T10:40:44,093 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:44,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46398 deadline: 1733740904089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:44,094 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:44,095 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:44,095 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:44,100 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:44,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46374 deadline: 1733740904099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:44,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:44,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:44,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:44,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:44,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46396 deadline: 1733740904100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:44,102 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:44,102 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:44,102 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:44,161 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:44,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46390 deadline: 1733740904159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:44,162 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:44,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46362 deadline: 1733740904159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:44,165 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 7ea43b39fc24,36101,1733740827678 for 'test_row_0' in 1d9d6bba7dcd09663c7b08474fc1c1b5 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2157 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T10:40:44,165 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 7ea43b39fc24,36101,1733740827678 for 'test_row_1' in 1d9d6bba7dcd09663c7b08474fc1c1b5 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2155 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T10:40:44,167 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:44,167 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:44,167 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:44,167 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:44,167 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:44,167 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:44,168 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=341 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/C/08070b46c9c94c2e978a3ade9226f7a1 2024-12-09T10:40:44,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/A/0135ce2f1414452eaa260c2232a8d4d7 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/0135ce2f1414452eaa260c2232a8d4d7 2024-12-09T10:40:44,184 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/0135ce2f1414452eaa260c2232a8d4d7, entries=150, sequenceid=341, filesize=12.0 K 2024-12-09T10:40:44,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/B/e30a28c65b274a589bb52c39d979de47 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/e30a28c65b274a589bb52c39d979de47 2024-12-09T10:40:44,192 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/e30a28c65b274a589bb52c39d979de47, entries=150, sequenceid=341, filesize=12.0 K 2024-12-09T10:40:44,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/C/08070b46c9c94c2e978a3ade9226f7a1 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/08070b46c9c94c2e978a3ade9226f7a1 2024-12-09T10:40:44,199 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/08070b46c9c94c2e978a3ade9226f7a1, entries=150, sequenceid=341, filesize=12.0 K 2024-12-09T10:40:44,200 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 1d9d6bba7dcd09663c7b08474fc1c1b5 in 1291ms, sequenceid=341, compaction requested=false 2024-12-09T10:40:44,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 1d9d6bba7dcd09663c7b08474fc1c1b5: 2024-12-09T10:40:44,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:44,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-12-09T10:40:44,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-12-09T10:40:44,206 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-12-09T10:40:44,206 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3830 sec 2024-12-09T10:40:44,210 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 2.3920 sec 2024-12-09T10:40:45,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(8855): Flush requested on 1d9d6bba7dcd09663c7b08474fc1c1b5 2024-12-09T10:40:45,103 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1d9d6bba7dcd09663c7b08474fc1c1b5 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-09T10:40:45,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d9d6bba7dcd09663c7b08474fc1c1b5, store=A 2024-12-09T10:40:45,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:40:45,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d9d6bba7dcd09663c7b08474fc1c1b5, store=B 2024-12-09T10:40:45,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:40:45,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d9d6bba7dcd09663c7b08474fc1c1b5, store=C 2024-12-09T10:40:45,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:40:45,111 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/A/89eb6028a0b5476889dfe3cf71258685 is 50, key is test_row_0/A:col10/1733740842948/Put/seqid=0 2024-12-09T10:40:45,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741903_1079 (size=14741) 2024-12-09T10:40:45,126 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=358 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/A/89eb6028a0b5476889dfe3cf71258685 2024-12-09T10:40:45,139 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/B/e09aaafaf9e34a5c9a65f13a3945bf0f is 50, key is test_row_0/B:col10/1733740842948/Put/seqid=0 2024-12-09T10:40:45,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741904_1080 (size=12301) 2024-12-09T10:40:45,150 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:45,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46374 deadline: 1733740905140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:45,151 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:45,151 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:45,151 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:45,156 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:45,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46396 deadline: 1733740905151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:45,157 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:45,157 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:45,157 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:45,157 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:45,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46398 deadline: 1733740905151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:45,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:45,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:45,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:45,262 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:45,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46374 deadline: 1733740905260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:45,263 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:45,263 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:45,263 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:45,263 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:45,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46396 deadline: 1733740905260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:45,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:45,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:45,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:45,277 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:45,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46398 deadline: 1733740905270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:45,279 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:45,279 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:45,279 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:45,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:45,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46374 deadline: 1733740905470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:45,471 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:45,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46396 deadline: 1733740905470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:45,471 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:45,471 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:45,471 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:45,471 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:45,471 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:45,472 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:45,494 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:45,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46398 deadline: 1733740905489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:45,495 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:45,495 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:45,495 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:45,547 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=358 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/B/e09aaafaf9e34a5c9a65f13a3945bf0f 2024-12-09T10:40:45,557 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/C/15d8ad392d81432a8e4928a4741ec520 is 50, key is test_row_0/C:col10/1733740842948/Put/seqid=0 2024-12-09T10:40:45,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741905_1081 (size=12301) 2024-12-09T10:40:45,781 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:45,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46374 deadline: 1733740905780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:45,781 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:45,781 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:45,781 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:45,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46396 deadline: 1733740905780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:45,781 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:45,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:45,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:45,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:45,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:45,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46398 deadline: 1733740905800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:45,803 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:45,803 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:45,803 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:45,963 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=358 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/C/15d8ad392d81432a8e4928a4741ec520 2024-12-09T10:40:45,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-09T10:40:45,970 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T10:40:45,970 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-09T10:40:45,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-12-09T10:40:45,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-09T10:40:45,975 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T10:40:45,976 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T10:40:45,976 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T10:40:45,976 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/A/89eb6028a0b5476889dfe3cf71258685 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/89eb6028a0b5476889dfe3cf71258685 2024-12-09T10:40:45,988 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/89eb6028a0b5476889dfe3cf71258685, entries=200, sequenceid=358, filesize=14.4 K 2024-12-09T10:40:45,990 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/B/e09aaafaf9e34a5c9a65f13a3945bf0f as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/e09aaafaf9e34a5c9a65f13a3945bf0f 2024-12-09T10:40:45,999 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/e09aaafaf9e34a5c9a65f13a3945bf0f, entries=150, sequenceid=358, filesize=12.0 K 2024-12-09T10:40:46,002 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/C/15d8ad392d81432a8e4928a4741ec520 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/15d8ad392d81432a8e4928a4741ec520 2024-12-09T10:40:46,012 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/15d8ad392d81432a8e4928a4741ec520, entries=150, sequenceid=358, filesize=12.0 K 2024-12-09T10:40:46,013 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 1d9d6bba7dcd09663c7b08474fc1c1b5 in 911ms, sequenceid=358, compaction requested=true 2024-12-09T10:40:46,013 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1d9d6bba7dcd09663c7b08474fc1c1b5: 2024-12-09T10:40:46,014 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T10:40:46,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1d9d6bba7dcd09663c7b08474fc1c1b5:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T10:40:46,015 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:40:46,015 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T10:40:46,015 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1d9d6bba7dcd09663c7b08474fc1c1b5:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T10:40:46,015 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40025 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T10:40:46,015 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:40:46,016 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1d9d6bba7dcd09663c7b08474fc1c1b5:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T10:40:46,016 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T10:40:46,016 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1541): 1d9d6bba7dcd09663c7b08474fc1c1b5/A is initiating minor compaction (all files) 2024-12-09T10:40:46,016 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1d9d6bba7dcd09663c7b08474fc1c1b5/A in TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:46,016 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/4f52605b6b144183b8544119e0e14e12, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/0135ce2f1414452eaa260c2232a8d4d7, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/89eb6028a0b5476889dfe3cf71258685] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp, totalSize=39.1 K 2024-12-09T10:40:46,017 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4f52605b6b144183b8544119e0e14e12, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1733740841889 2024-12-09T10:40:46,017 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T10:40:46,017 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0135ce2f1414452eaa260c2232a8d4d7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1733740842286 2024-12-09T10:40:46,017 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1541): 1d9d6bba7dcd09663c7b08474fc1c1b5/B is initiating minor compaction (all files) 2024-12-09T10:40:46,017 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1d9d6bba7dcd09663c7b08474fc1c1b5/B in TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:46,017 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/b3656d6396a94ffdb3e4ce3b0750b9b3, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/e30a28c65b274a589bb52c39d979de47, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/e09aaafaf9e34a5c9a65f13a3945bf0f] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp, totalSize=36.7 K 2024-12-09T10:40:46,019 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 89eb6028a0b5476889dfe3cf71258685, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1733740842944 2024-12-09T10:40:46,019 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting b3656d6396a94ffdb3e4ce3b0750b9b3, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1733740841889 2024-12-09T10:40:46,020 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting e30a28c65b274a589bb52c39d979de47, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1733740842286 2024-12-09T10:40:46,022 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting e09aaafaf9e34a5c9a65f13a3945bf0f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1733740842948 2024-12-09T10:40:46,039 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1d9d6bba7dcd09663c7b08474fc1c1b5#A#compaction#69 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:40:46,039 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/A/f31656a500b14b389dbeea2c912cb547 is 50, key is test_row_0/A:col10/1733740842948/Put/seqid=0 2024-12-09T10:40:46,042 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1d9d6bba7dcd09663c7b08474fc1c1b5#B#compaction#70 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:40:46,043 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/B/a55a0c1d121a4b218bce2d5860448c6e is 50, key is test_row_0/B:col10/1733740842948/Put/seqid=0 2024-12-09T10:40:46,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741907_1083 (size=13085) 2024-12-09T10:40:46,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741906_1082 (size=13085) 2024-12-09T10:40:46,074 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/B/a55a0c1d121a4b218bce2d5860448c6e as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/a55a0c1d121a4b218bce2d5860448c6e 2024-12-09T10:40:46,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-09T10:40:46,082 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/A/f31656a500b14b389dbeea2c912cb547 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/f31656a500b14b389dbeea2c912cb547 2024-12-09T10:40:46,089 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1d9d6bba7dcd09663c7b08474fc1c1b5/B of 1d9d6bba7dcd09663c7b08474fc1c1b5 into a55a0c1d121a4b218bce2d5860448c6e(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:40:46,089 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1d9d6bba7dcd09663c7b08474fc1c1b5: 2024-12-09T10:40:46,090 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., storeName=1d9d6bba7dcd09663c7b08474fc1c1b5/B, priority=13, startTime=1733740846015; duration=0sec 2024-12-09T10:40:46,090 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T10:40:46,090 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1d9d6bba7dcd09663c7b08474fc1c1b5:B 2024-12-09T10:40:46,091 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T10:40:46,092 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1d9d6bba7dcd09663c7b08474fc1c1b5/A of 1d9d6bba7dcd09663c7b08474fc1c1b5 into f31656a500b14b389dbeea2c912cb547(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:40:46,092 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1d9d6bba7dcd09663c7b08474fc1c1b5: 2024-12-09T10:40:46,092 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., storeName=1d9d6bba7dcd09663c7b08474fc1c1b5/A, priority=13, startTime=1733740846013; duration=0sec 2024-12-09T10:40:46,092 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:40:46,092 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1d9d6bba7dcd09663c7b08474fc1c1b5:A 2024-12-09T10:40:46,092 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T10:40:46,093 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1541): 1d9d6bba7dcd09663c7b08474fc1c1b5/C is initiating minor compaction (all files) 2024-12-09T10:40:46,093 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1d9d6bba7dcd09663c7b08474fc1c1b5/C in TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:46,093 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/75e17d09da42431cb6836abb3a9a2323, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/08070b46c9c94c2e978a3ade9226f7a1, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/15d8ad392d81432a8e4928a4741ec520] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp, totalSize=36.7 K 2024-12-09T10:40:46,093 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 75e17d09da42431cb6836abb3a9a2323, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1733740841889 2024-12-09T10:40:46,094 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 08070b46c9c94c2e978a3ade9226f7a1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1733740842286 2024-12-09T10:40:46,094 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 15d8ad392d81432a8e4928a4741ec520, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1733740842948 2024-12-09T10:40:46,117 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1d9d6bba7dcd09663c7b08474fc1c1b5#C#compaction#71 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:40:46,117 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/C/29e99b2a489b4f07b3af68651885ba83 is 50, key is test_row_0/C:col10/1733740842948/Put/seqid=0 2024-12-09T10:40:46,129 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-09T10:40:46,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:46,129 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 1d9d6bba7dcd09663c7b08474fc1c1b5 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-09T10:40:46,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d9d6bba7dcd09663c7b08474fc1c1b5, store=A 2024-12-09T10:40:46,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:40:46,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d9d6bba7dcd09663c7b08474fc1c1b5, store=B 2024-12-09T10:40:46,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:40:46,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d9d6bba7dcd09663c7b08474fc1c1b5, store=C 2024-12-09T10:40:46,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:40:46,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/A/9f750b17c02646248adefcbe4d6414b9 is 50, key is test_row_0/A:col10/1733740845137/Put/seqid=0 2024-12-09T10:40:46,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741908_1084 (size=13085) 2024-12-09T10:40:46,158 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/C/29e99b2a489b4f07b3af68651885ba83 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/29e99b2a489b4f07b3af68651885ba83 2024-12-09T10:40:46,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741909_1085 (size=12301) 2024-12-09T10:40:46,159 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=378 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/A/9f750b17c02646248adefcbe4d6414b9 2024-12-09T10:40:46,171 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1d9d6bba7dcd09663c7b08474fc1c1b5/C of 1d9d6bba7dcd09663c7b08474fc1c1b5 into 29e99b2a489b4f07b3af68651885ba83(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:40:46,171 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1d9d6bba7dcd09663c7b08474fc1c1b5: 2024-12-09T10:40:46,171 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., storeName=1d9d6bba7dcd09663c7b08474fc1c1b5/C, priority=13, startTime=1733740846015; duration=0sec 2024-12-09T10:40:46,171 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:40:46,172 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1d9d6bba7dcd09663c7b08474fc1c1b5:C 2024-12-09T10:40:46,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/B/219d57041a124d99b5b256375c467776 is 50, key is test_row_0/B:col10/1733740845137/Put/seqid=0 2024-12-09T10:40:46,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(8855): Flush requested on 1d9d6bba7dcd09663c7b08474fc1c1b5 2024-12-09T10:40:46,183 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. as already flushing 2024-12-09T10:40:46,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741910_1086 (size=12301) 2024-12-09T10:40:46,220 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:46,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46362 deadline: 1733740906214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:46,220 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:46,220 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:46,221 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:46,221 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:46,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46390 deadline: 1733740906216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:46,222 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:46,222 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:46,222 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:46,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-09T10:40:46,294 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:46,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46396 deadline: 1733740906291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:46,295 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:46,295 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:46,295 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:46,295 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:46,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46374 deadline: 1733740906291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:46,296 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:46,296 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:46,296 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:46,313 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:46,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46398 deadline: 1733740906309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:46,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:46,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:46,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:46,330 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:46,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46390 deadline: 1733740906330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:46,331 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:46,331 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:46,331 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:46,331 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:46,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46362 deadline: 1733740906330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:46,332 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:46,332 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:46,332 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:46,543 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:46,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46390 deadline: 1733740906540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:46,544 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:46,544 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:46,544 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:46,544 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:46,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46362 deadline: 1733740906540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:46,544 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:46,544 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:46,544 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:46,596 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=378 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/B/219d57041a124d99b5b256375c467776 2024-12-09T10:40:46,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-09T10:40:46,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/C/60f6a62fdc5645fe8fb4fd1760787726 is 50, key is test_row_0/C:col10/1733740845137/Put/seqid=0 2024-12-09T10:40:46,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741911_1087 (size=12301) 2024-12-09T10:40:46,854 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:46,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46362 deadline: 1733740906849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:46,855 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:46,855 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:46,855 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:46,855 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:46,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46390 deadline: 1733740906850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:46,855 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:46,855 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:46,855 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:47,011 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=378 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/C/60f6a62fdc5645fe8fb4fd1760787726 2024-12-09T10:40:47,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/A/9f750b17c02646248adefcbe4d6414b9 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/9f750b17c02646248adefcbe4d6414b9 2024-12-09T10:40:47,034 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/9f750b17c02646248adefcbe4d6414b9, entries=150, sequenceid=378, filesize=12.0 K 2024-12-09T10:40:47,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/B/219d57041a124d99b5b256375c467776 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/219d57041a124d99b5b256375c467776 2024-12-09T10:40:47,042 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/219d57041a124d99b5b256375c467776, entries=150, sequenceid=378, filesize=12.0 K 2024-12-09T10:40:47,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/C/60f6a62fdc5645fe8fb4fd1760787726 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/60f6a62fdc5645fe8fb4fd1760787726 2024-12-09T10:40:47,050 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/60f6a62fdc5645fe8fb4fd1760787726, entries=150, sequenceid=378, filesize=12.0 K 2024-12-09T10:40:47,051 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 1d9d6bba7dcd09663c7b08474fc1c1b5 in 922ms, sequenceid=378, compaction requested=false 2024-12-09T10:40:47,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 1d9d6bba7dcd09663c7b08474fc1c1b5: 2024-12-09T10:40:47,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:47,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-12-09T10:40:47,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-12-09T10:40:47,060 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-12-09T10:40:47,060 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0780 sec 2024-12-09T10:40:47,063 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.0910 sec 2024-12-09T10:40:47,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-09T10:40:47,110 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T10:40:47,111 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-09T10:40:47,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-12-09T10:40:47,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-09T10:40:47,114 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T10:40:47,115 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T10:40:47,115 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T10:40:47,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-09T10:40:47,269 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-09T10:40:47,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:47,269 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 1d9d6bba7dcd09663c7b08474fc1c1b5 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-09T10:40:47,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d9d6bba7dcd09663c7b08474fc1c1b5, store=A 2024-12-09T10:40:47,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:40:47,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d9d6bba7dcd09663c7b08474fc1c1b5, store=B 2024-12-09T10:40:47,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:40:47,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d9d6bba7dcd09663c7b08474fc1c1b5, store=C 2024-12-09T10:40:47,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:40:47,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/A/69c31daaceea4851bd3a1a7a20e35ab3 is 50, key is test_row_0/A:col10/1733740846214/Put/seqid=0 2024-12-09T10:40:47,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(8855): Flush requested on 1d9d6bba7dcd09663c7b08474fc1c1b5 2024-12-09T10:40:47,312 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. as already flushing 2024-12-09T10:40:47,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741912_1088 (size=12301) 2024-12-09T10:40:47,329 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=393 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/A/69c31daaceea4851bd3a1a7a20e35ab3 2024-12-09T10:40:47,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/B/d7560fda3d2d4d08b2cef8e478502c82 is 50, key is test_row_0/B:col10/1733740846214/Put/seqid=0 2024-12-09T10:40:47,382 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:47,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46374 deadline: 1733740907367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:47,383 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:47,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46362 deadline: 1733740907370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:47,383 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:47,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46390 deadline: 1733740907370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:47,384 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:47,421 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:47,421 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:47,421 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:47,421 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:47,421 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:47,422 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:47,422 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:47,388 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:47,387 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:47,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46396 deadline: 1733740907383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:47,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46398 deadline: 1733740907382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:47,422 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:47,423 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:47,423 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:47,423 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:47,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741913_1089 (size=12301) 2024-12-09T10:40:47,424 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=393 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/B/d7560fda3d2d4d08b2cef8e478502c82 2024-12-09T10:40:47,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-09T10:40:47,431 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:47,431 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:47,431 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:47,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/C/19422dcb1d1946d0afabfa1e18151058 is 50, key is test_row_0/C:col10/1733740846214/Put/seqid=0 2024-12-09T10:40:47,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741914_1090 (size=12301) 2024-12-09T10:40:47,468 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=393 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/C/19422dcb1d1946d0afabfa1e18151058 2024-12-09T10:40:47,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/A/69c31daaceea4851bd3a1a7a20e35ab3 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/69c31daaceea4851bd3a1a7a20e35ab3 2024-12-09T10:40:47,498 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/69c31daaceea4851bd3a1a7a20e35ab3, entries=150, sequenceid=393, filesize=12.0 K 2024-12-09T10:40:47,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/B/d7560fda3d2d4d08b2cef8e478502c82 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/d7560fda3d2d4d08b2cef8e478502c82 2024-12-09T10:40:47,507 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/d7560fda3d2d4d08b2cef8e478502c82, entries=150, sequenceid=393, filesize=12.0 K 2024-12-09T10:40:47,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/C/19422dcb1d1946d0afabfa1e18151058 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/19422dcb1d1946d0afabfa1e18151058 2024-12-09T10:40:47,523 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/19422dcb1d1946d0afabfa1e18151058, entries=150, sequenceid=393, filesize=12.0 K 2024-12-09T10:40:47,524 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for 1d9d6bba7dcd09663c7b08474fc1c1b5 in 255ms, sequenceid=393, compaction requested=true 2024-12-09T10:40:47,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 1d9d6bba7dcd09663c7b08474fc1c1b5: 2024-12-09T10:40:47,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:47,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-12-09T10:40:47,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-12-09T10:40:47,531 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-12-09T10:40:47,531 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 412 msec 2024-12-09T10:40:47,533 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 421 msec 2024-12-09T10:40:47,542 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1733740847365, max timestamp=1733740847539Num uniques 859; 2024-12-09T10:40:47,542 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T10:40:47,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(8855): Flush requested on 1d9d6bba7dcd09663c7b08474fc1c1b5 2024-12-09T10:40:47,543 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1d9d6bba7dcd09663c7b08474fc1c1b5 3/3 column families, dataSize=91.69 KB heapSize=267.19 KB 2024-12-09T10:40:47,544 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1733740847365, max timestamp=1733740847539Num uniques 859; 2024-12-09T10:40:47,544 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T10:40:47,546 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733740847365, max timestamp=1733740847539Num uniques 859; 2024-12-09T10:40:47,546 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-09T10:40:47,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d9d6bba7dcd09663c7b08474fc1c1b5, store=A 2024-12-09T10:40:47,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T10:40:47,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d9d6bba7dcd09663c7b08474fc1c1b5, store=B 2024-12-09T10:40:47,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T10:40:47,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d9d6bba7dcd09663c7b08474fc1c1b5, store=C 2024-12-09T10:40:47,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T10:40:47,568 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/A/74288a0f7b024167a0e4b144ebd5f31b is 50, key is test_row_0/A:col10/1733740847379/Put/seqid=0 2024-12-09T10:40:47,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741915_1091 (size=12301) 2024-12-09T10:40:47,650 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733740847609, max timestamp=1733740847645Num uniques 859; 2024-12-09T10:40:47,650 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733740847609, max timestamp=1733740847645Num uniques 859; 2024-12-09T10:40:47,650 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T10:40:47,650 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T10:40:47,650 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733740847609, max timestamp=1733740847646Num uniques 859; 2024-12-09T10:40:47,650 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T10:40:47,726 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1733740847692, max timestamp=1733740847720Num uniques 1018; 2024-12-09T10:40:47,727 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1733740847692, max timestamp=1733740847720Num uniques 1018; 2024-12-09T10:40:47,727 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=14558, suffix heap size=133840,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=318 2024-12-09T10:40:47,727 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=14558, suffix heap size=133840,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=318 2024-12-09T10:40:47,727 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1733740847692, max timestamp=1733740847720Num uniques 1018; 2024-12-09T10:40:47,727 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=14558, suffix heap size=133840,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=318 2024-12-09T10:40:47,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-09T10:40:47,740 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T10:40:47,741 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-09T10:40:47,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-12-09T10:40:47,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-09T10:40:47,745 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T10:40:47,747 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T10:40:47,748 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T10:40:47,791 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:47,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46374 deadline: 1733740907785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:47,792 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:47,792 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:47,792 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:47,794 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733740847767, max timestamp=1733740847785Num uniques 1177; 2024-12-09T10:40:47,795 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=53902, new segment data size=8102, suffix heap size=150376,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1177, new segment cells count=177 2024-12-09T10:40:47,795 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733740847767, max timestamp=1733740847785Num uniques 1177; 2024-12-09T10:40:47,796 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=53902, new segment data size=8102, suffix heap size=150376,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1177, new segment cells count=177 2024-12-09T10:40:47,797 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733740847767, max timestamp=1733740847785Num uniques 1177; 2024-12-09T10:40:47,797 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=53902, new segment data size=8102, suffix heap size=150376,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1177, new segment cells count=177 2024-12-09T10:40:47,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-09T10:40:47,868 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733740847842, max timestamp=1733740847861Num uniques 1036; 2024-12-09T10:40:47,869 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-12-09T10:40:47,869 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733740847842, max timestamp=1733740847861Num uniques 1036; 2024-12-09T10:40:47,870 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-12-09T10:40:47,873 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733740847842, max timestamp=1733740847861Num uniques 1036; 2024-12-09T10:40:47,873 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-12-09T10:40:47,901 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-09T10:40:47,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:47,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. as already flushing 2024-12-09T10:40:47,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:47,901 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:40:47,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:40:47,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:40:47,930 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1733740847908, max timestamp=1733740847924Num uniques 1045; 2024-12-09T10:40:47,930 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1733740847908, max timestamp=1733740847924Num uniques 1045; 2024-12-09T10:40:47,931 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=13510, suffix heap size=136648,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=295 2024-12-09T10:40:47,931 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=13510, suffix heap size=136648,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=295 2024-12-09T10:40:47,931 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1733740847908, max timestamp=1733740847924Num uniques 1045; 2024-12-09T10:40:47,931 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=13510, suffix heap size=136648,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=295 2024-12-09T10:40:47,980 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=254, cellSize=11630, totalHeapSize=26656, min timestamp=1733740847964, max timestamp=1733740847976Num uniques 1154; 2024-12-09T10:40:47,980 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=254, cellSize=11630, totalHeapSize=26656, min timestamp=1733740847964, max timestamp=1733740847976Num uniques 1154; 2024-12-09T10:40:47,980 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=52850, new segment data size=11630, suffix heap size=147984,new segment heap size=26656 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1154, new segment cells count=254 2024-12-09T10:40:47,980 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=52850, new segment data size=11630, suffix heap size=147984,new segment heap size=26656 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1154, new segment cells count=254 2024-12-09T10:40:47,984 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=254, cellSize=11630, totalHeapSize=26656, min timestamp=1733740847964, max timestamp=1733740847976Num uniques 1154; 2024-12-09T10:40:47,984 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=52850, new segment data size=11630, suffix heap size=147984,new segment heap size=26656 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1154, new segment cells count=254 2024-12-09T10:40:48,013 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=414 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/A/74288a0f7b024167a0e4b144ebd5f31b 2024-12-09T10:40:48,040 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/B/46a8bb2c8759422e932e70535e94a503 is 50, key is test_row_0/B:col10/1733740847379/Put/seqid=0 2024-12-09T10:40:48,055 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=313, cellSize=14328, totalHeapSize=32792, min timestamp=1733740848018, max timestamp=1733740848051Num uniques 1113; 2024-12-09T10:40:48,055 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=50968, new segment data size=14328, suffix heap size=143720,new segment heap size=32792 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1113, new segment cells count=313 2024-12-09T10:40:48,057 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-09T10:40:48,060 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=313, cellSize=14328, totalHeapSize=32792, min timestamp=1733740848018, max timestamp=1733740848051Num uniques 1113; 2024-12-09T10:40:48,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-09T10:40:48,060 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=50968, new segment data size=14328, suffix heap size=143720,new segment heap size=32792 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1113, new segment cells count=313 2024-12-09T10:40:48,063 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=313, cellSize=14328, totalHeapSize=32792, min timestamp=1733740848018, max timestamp=1733740848051Num uniques 1113; 2024-12-09T10:40:48,064 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=50968, new segment data size=14328, suffix heap size=143720,new segment heap size=32792 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1113, new segment cells count=313 2024-12-09T10:40:48,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:48,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. as already flushing 2024-12-09T10:40:48,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:48,074 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:40:48,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:40:48,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:40:48,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741916_1092 (size=12301) 2024-12-09T10:40:48,155 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=322, cellSize=14742, totalHeapSize=33728, min timestamp=1733740848073, max timestamp=1733740848151Num uniques 1172; 2024-12-09T10:40:48,155 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=53672, new segment data size=14742, suffix heap size=149856,new segment heap size=33728 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1172, new segment cells count=322 2024-12-09T10:40:48,159 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=322, cellSize=14742, totalHeapSize=33728, min timestamp=1733740848073, max timestamp=1733740848151Num uniques 1172; 2024-12-09T10:40:48,159 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=53672, new segment data size=14742, suffix heap size=149856,new segment heap size=33728 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1172, new segment cells count=322 2024-12-09T10:40:48,159 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=322, cellSize=14742, totalHeapSize=33728, min timestamp=1733740848073, max timestamp=1733740848151Num uniques 1172; 2024-12-09T10:40:48,160 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=53672, new segment data size=14742, suffix heap size=149856,new segment heap size=33728 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1172, new segment cells count=322 2024-12-09T10:40:48,240 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=331, cellSize=15156, totalHeapSize=34664, min timestamp=1733740848202, max timestamp=1733740848235Num uniques 1181; 2024-12-09T10:40:48,240 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=54086, new segment data size=15156, suffix heap size=150792,new segment heap size=34664 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1181, new segment cells count=331 2024-12-09T10:40:48,240 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=331, cellSize=15156, totalHeapSize=34664, min timestamp=1733740848202, max timestamp=1733740848235Num uniques 1181; 2024-12-09T10:40:48,240 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=54086, new segment data size=15156, suffix heap size=150792,new segment heap size=34664 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1181, new segment cells count=331 2024-12-09T10:40:48,240 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=331, cellSize=15156, totalHeapSize=34664, min timestamp=1733740848202, max timestamp=1733740848235Num uniques 1181; 2024-12-09T10:40:48,241 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=54086, new segment data size=15156, suffix heap size=150792,new segment heap size=34664 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1181, new segment cells count=331 2024-12-09T10:40:48,249 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-09T10:40:48,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:48,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. as already flushing 2024-12-09T10:40:48,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:48,249 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:40:48,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:40:48,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:40:48,317 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=340, cellSize=15570, totalHeapSize=35600, min timestamp=1733740848273, max timestamp=1733740848312Num uniques 1190; 2024-12-09T10:40:48,317 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=340, cellSize=15570, totalHeapSize=35600, min timestamp=1733740848273, max timestamp=1733740848312Num uniques 1190; 2024-12-09T10:40:48,317 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=54500, new segment data size=15570, suffix heap size=151728,new segment heap size=35600 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1190, new segment cells count=340 2024-12-09T10:40:48,317 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=54500, new segment data size=15570, suffix heap size=151728,new segment heap size=35600 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1190, new segment cells count=340 2024-12-09T10:40:48,318 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=340, cellSize=15570, totalHeapSize=35600, min timestamp=1733740848273, max timestamp=1733740848312Num uniques 1190; 2024-12-09T10:40:48,318 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=54500, new segment data size=15570, suffix heap size=151728,new segment heap size=35600 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1190, new segment cells count=340 2024-12-09T10:40:48,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-09T10:40:48,404 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-09T10:40:48,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:48,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. as already flushing 2024-12-09T10:40:48,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:48,406 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:40:48,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:40:48,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:40:48,435 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=299, cellSize=13694, totalHeapSize=31336, min timestamp=1733740848401, max timestamp=1733740848427Num uniques 1199; 2024-12-09T10:40:48,435 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=299, cellSize=13694, totalHeapSize=31336, min timestamp=1733740848401, max timestamp=1733740848427Num uniques 1199; 2024-12-09T10:40:48,435 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=54914, new segment data size=13694, suffix heap size=152664,new segment heap size=31336 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1199, new segment cells count=299 2024-12-09T10:40:48,435 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=54914, new segment data size=13694, suffix heap size=152664,new segment heap size=31336 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1199, new segment cells count=299 2024-12-09T10:40:48,435 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=299, cellSize=13694, totalHeapSize=31336, min timestamp=1733740848401, max timestamp=1733740848427Num uniques 1199; 2024-12-09T10:40:48,435 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=54914, new segment data size=13694, suffix heap size=152664,new segment heap size=31336 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1199, new segment cells count=299 2024-12-09T10:40:48,478 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=208, cellSize=9520, totalHeapSize=21872, min timestamp=1733740848456, max timestamp=1733740848473Num uniques 1158; 2024-12-09T10:40:48,478 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=53030, new segment data size=9520, suffix heap size=148400,new segment heap size=21872 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1158, new segment cells count=208 2024-12-09T10:40:48,478 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=208, cellSize=9520, totalHeapSize=21872, min timestamp=1733740848456, max timestamp=1733740848473Num uniques 1158; 2024-12-09T10:40:48,479 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=53030, new segment data size=9520, suffix heap size=148400,new segment heap size=21872 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1158, new segment cells count=208 2024-12-09T10:40:48,480 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=158, cellSize=7230, totalHeapSize=16672, min timestamp=1733740848456, max timestamp=1733740848477Num uniques 1158; 2024-12-09T10:40:48,480 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=53030, new segment data size=7230, suffix heap size=148400,new segment heap size=16672 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1158, new segment cells count=158 2024-12-09T10:40:48,489 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=414 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/B/46a8bb2c8759422e932e70535e94a503 2024-12-09T10:40:48,515 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/C/dae218112e1c49df9c43df657b5934f1 is 50, key is test_row_0/C:col10/1733740847379/Put/seqid=0 2024-12-09T10:40:48,526 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=167, cellSize=7635, totalHeapSize=17608, min timestamp=1733740848501, max timestamp=1733740848522Num uniques 1067; 2024-12-09T10:40:48,526 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48855, new segment data size=7635, suffix heap size=138936,new segment heap size=17608 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1067, new segment cells count=167 2024-12-09T10:40:48,526 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=167, cellSize=7642, totalHeapSize=17608, min timestamp=1733740848501, max timestamp=1733740848521Num uniques 1017; 2024-12-09T10:40:48,526 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46572, new segment data size=7642, suffix heap size=133736,new segment heap size=17608 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1017, new segment cells count=167 2024-12-09T10:40:48,530 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=155, cellSize=7095, totalHeapSize=16360, min timestamp=1733740848501, max timestamp=1733740848522Num uniques 1067; 2024-12-09T10:40:48,530 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48857, new segment data size=7095, suffix heap size=138936,new segment heap size=16360 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1067, new segment cells count=155 2024-12-09T10:40:48,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741917_1093 (size=12301) 2024-12-09T10:40:48,540 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.55 KB at sequenceid=414 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/C/dae218112e1c49df9c43df657b5934f1 2024-12-09T10:40:48,550 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/A/74288a0f7b024167a0e4b144ebd5f31b as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/74288a0f7b024167a0e4b144ebd5f31b 2024-12-09T10:40:48,559 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/74288a0f7b024167a0e4b144ebd5f31b, entries=150, sequenceid=414, filesize=12.0 K 2024-12-09T10:40:48,560 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-09T10:40:48,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:48,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. as already flushing 2024-12-09T10:40:48,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:48,561 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:40:48,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:40:48,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:40:48,567 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/B/46a8bb2c8759422e932e70535e94a503 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/46a8bb2c8759422e932e70535e94a503 2024-12-09T10:40:48,577 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=226, cellSize=10346, totalHeapSize=23744, min timestamp=1733740848543, max timestamp=1733740848573Num uniques 1026; 2024-12-09T10:40:48,577 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46986, new segment data size=10346, suffix heap size=134672,new segment heap size=23744 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1026, new segment cells count=226 2024-12-09T10:40:48,577 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=226, cellSize=10346, totalHeapSize=23744, min timestamp=1733740848543, max timestamp=1733740848573Num uniques 1026; 2024-12-09T10:40:48,577 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=226, cellSize=10346, totalHeapSize=23744, min timestamp=1733740848543, max timestamp=1733740848573Num uniques 1014; 2024-12-09T10:40:48,578 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46986, new segment data size=10346, suffix heap size=134672,new segment heap size=23744 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1026, new segment cells count=226 2024-12-09T10:40:48,578 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46444, new segment data size=10346, suffix heap size=133424,new segment heap size=23744 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1014, new segment cells count=226 2024-12-09T10:40:48,591 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/46a8bb2c8759422e932e70535e94a503, entries=150, sequenceid=414, filesize=12.0 K 2024-12-09T10:40:48,592 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/C/dae218112e1c49df9c43df657b5934f1 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/dae218112e1c49df9c43df657b5934f1 2024-12-09T10:40:48,599 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/dae218112e1c49df9c43df657b5934f1, entries=150, sequenceid=414, filesize=12.0 K 2024-12-09T10:40:48,628 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~30.91 KB/31655, heapSize ~75.43 KB/77240, currentSize=60.38 KB/61830 for 1d9d6bba7dcd09663c7b08474fc1c1b5 in 1085ms, sequenceid=414, compaction requested=true 2024-12-09T10:40:48,629 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1d9d6bba7dcd09663c7b08474fc1c1b5: 2024-12-09T10:40:48,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(8855): Flush requested on 1d9d6bba7dcd09663c7b08474fc1c1b5 2024-12-09T10:40:48,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1d9d6bba7dcd09663c7b08474fc1c1b5:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T10:40:48,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:40:48,629 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T10:40:48,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1d9d6bba7dcd09663c7b08474fc1c1b5:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T10:40:48,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T10:40:48,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1d9d6bba7dcd09663c7b08474fc1c1b5:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T10:40:48,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T10:40:48,629 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T10:40:48,630 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1d9d6bba7dcd09663c7b08474fc1c1b5 3/3 column families, dataSize=73.80 KB heapSize=199.41 KB 2024-12-09T10:40:48,631 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49988 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T10:40:48,631 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1541): 1d9d6bba7dcd09663c7b08474fc1c1b5/A is initiating minor compaction (all files) 2024-12-09T10:40:48,631 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1d9d6bba7dcd09663c7b08474fc1c1b5/A in TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:48,631 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49988 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T10:40:48,632 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1541): 1d9d6bba7dcd09663c7b08474fc1c1b5/B is initiating minor compaction (all files) 2024-12-09T10:40:48,632 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/f31656a500b14b389dbeea2c912cb547, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/9f750b17c02646248adefcbe4d6414b9, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/69c31daaceea4851bd3a1a7a20e35ab3, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/74288a0f7b024167a0e4b144ebd5f31b] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp, totalSize=48.8 K 2024-12-09T10:40:48,632 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1d9d6bba7dcd09663c7b08474fc1c1b5/B in TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:48,632 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/a55a0c1d121a4b218bce2d5860448c6e, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/219d57041a124d99b5b256375c467776, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/d7560fda3d2d4d08b2cef8e478502c82, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/46a8bb2c8759422e932e70535e94a503] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp, totalSize=48.8 K 2024-12-09T10:40:48,632 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting f31656a500b14b389dbeea2c912cb547, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1733740842948 2024-12-09T10:40:48,632 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting a55a0c1d121a4b218bce2d5860448c6e, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1733740842948 2024-12-09T10:40:48,633 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 219d57041a124d99b5b256375c467776, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1733740845135 2024-12-09T10:40:48,633 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9f750b17c02646248adefcbe4d6414b9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1733740845135 2024-12-09T10:40:48,633 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting d7560fda3d2d4d08b2cef8e478502c82, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1733740846208 2024-12-09T10:40:48,633 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 69c31daaceea4851bd3a1a7a20e35ab3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1733740846208 2024-12-09T10:40:48,634 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 46a8bb2c8759422e932e70535e94a503, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=414, earliestPutTs=1733740847379 2024-12-09T10:40:48,634 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 74288a0f7b024167a0e4b144ebd5f31b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=414, earliestPutTs=1733740847379 2024-12-09T10:40:48,634 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d9d6bba7dcd09663c7b08474fc1c1b5, store=A 2024-12-09T10:40:48,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T10:40:48,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d9d6bba7dcd09663c7b08474fc1c1b5, store=B 2024-12-09T10:40:48,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T10:40:48,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d9d6bba7dcd09663c7b08474fc1c1b5, store=C 2024-12-09T10:40:48,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T10:40:48,652 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/A/58b13ded44db4ccd8d78d4abdb8eac6b is 50, key is test_row_0/A:col10/1733740848629/Put/seqid=0 2024-12-09T10:40:48,660 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1d9d6bba7dcd09663c7b08474fc1c1b5#A#compaction#82 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:40:48,661 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/A/18757bffe85a4749931bd1bdf4978283 is 50, key is test_row_0/A:col10/1733740847379/Put/seqid=0 2024-12-09T10:40:48,676 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1d9d6bba7dcd09663c7b08474fc1c1b5#B#compaction#83 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:40:48,677 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/B/a076f9c734dc4963995bff79b107ce62 is 50, key is test_row_0/B:col10/1733740847379/Put/seqid=0 2024-12-09T10:40:48,704 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:48,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46390 deadline: 1733740908682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:48,706 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:48,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46374 deadline: 1733740908689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:48,707 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:48,707 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:48,707 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:48,707 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:48,707 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:48,707 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:48,712 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:48,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46396 deadline: 1733740908702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:48,712 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:48,712 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:48,712 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:48,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46362 deadline: 1733740908708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:48,712 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:48,713 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:48,713 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:48,713 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:48,714 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-09T10:40:48,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:48,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. as already flushing 2024-12-09T10:40:48,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:48,715 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:40:48,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:40:48,714 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:48,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:40:48,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46398 deadline: 1733740908709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:48,724 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:48,725 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:48,725 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:48,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741918_1094 (size=12301) 2024-12-09T10:40:48,733 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=667 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/A/58b13ded44db4ccd8d78d4abdb8eac6b 2024-12-09T10:40:48,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741920_1096 (size=13221) 2024-12-09T10:40:48,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741919_1095 (size=13221) 2024-12-09T10:40:48,770 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/B/5d32ca380ef443d8a047eeb675eab8e3 is 50, key is test_row_0/B:col10/1733740848629/Put/seqid=0 2024-12-09T10:40:48,781 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/A/18757bffe85a4749931bd1bdf4978283 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/18757bffe85a4749931bd1bdf4978283 2024-12-09T10:40:48,790 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1d9d6bba7dcd09663c7b08474fc1c1b5/A of 1d9d6bba7dcd09663c7b08474fc1c1b5 into 18757bffe85a4749931bd1bdf4978283(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:40:48,790 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1d9d6bba7dcd09663c7b08474fc1c1b5: 2024-12-09T10:40:48,790 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., storeName=1d9d6bba7dcd09663c7b08474fc1c1b5/A, priority=12, startTime=1733740848629; duration=0sec 2024-12-09T10:40:48,792 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T10:40:48,792 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1d9d6bba7dcd09663c7b08474fc1c1b5:A 2024-12-09T10:40:48,792 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T10:40:48,794 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49988 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T10:40:48,794 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1541): 1d9d6bba7dcd09663c7b08474fc1c1b5/C is initiating minor compaction (all files) 2024-12-09T10:40:48,795 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1d9d6bba7dcd09663c7b08474fc1c1b5/C in TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:48,795 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/29e99b2a489b4f07b3af68651885ba83, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/60f6a62fdc5645fe8fb4fd1760787726, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/19422dcb1d1946d0afabfa1e18151058, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/dae218112e1c49df9c43df657b5934f1] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp, totalSize=48.8 K 2024-12-09T10:40:48,795 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 29e99b2a489b4f07b3af68651885ba83, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1733740842948 2024-12-09T10:40:48,796 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 60f6a62fdc5645fe8fb4fd1760787726, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1733740845135 2024-12-09T10:40:48,797 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 19422dcb1d1946d0afabfa1e18151058, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1733740846208 2024-12-09T10:40:48,797 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting dae218112e1c49df9c43df657b5934f1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=414, earliestPutTs=1733740847379 2024-12-09T10:40:48,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741921_1097 (size=12301) 2024-12-09T10:40:48,812 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=667 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/B/5d32ca380ef443d8a047eeb675eab8e3 2024-12-09T10:40:48,817 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:48,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46374 deadline: 1733740908810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:48,818 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:48,818 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:48,818 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:48,818 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:48,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46390 deadline: 1733740908810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:48,819 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:48,819 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:48,819 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:48,828 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:48,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46362 deadline: 1733740908819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:48,829 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:48,829 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:48,829 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:48,829 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1d9d6bba7dcd09663c7b08474fc1c1b5#C#compaction#85 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:40:48,829 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:48,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46396 deadline: 1733740908820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:48,830 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/C/7c2d9230f5d04252b1c75918152a42ea is 50, key is test_row_0/C:col10/1733740847379/Put/seqid=0 2024-12-09T10:40:48,830 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:48,830 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:48,830 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:48,838 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:48,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46398 deadline: 1733740908830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:48,840 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/C/3f82dbf5e17c42ef85929e8237c0bb7e is 50, key is test_row_0/C:col10/1733740848629/Put/seqid=0 2024-12-09T10:40:48,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:48,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:48,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:48,878 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-09T10:40:48,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:48,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. as already flushing 2024-12-09T10:40:48,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:48,879 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:40:48,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:40:48,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:40:48,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-09T10:40:48,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741922_1098 (size=13221) 2024-12-09T10:40:48,892 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/C/7c2d9230f5d04252b1c75918152a42ea as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/7c2d9230f5d04252b1c75918152a42ea 2024-12-09T10:40:48,901 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1d9d6bba7dcd09663c7b08474fc1c1b5/C of 1d9d6bba7dcd09663c7b08474fc1c1b5 into 7c2d9230f5d04252b1c75918152a42ea(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:40:48,901 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1d9d6bba7dcd09663c7b08474fc1c1b5: 2024-12-09T10:40:48,901 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., storeName=1d9d6bba7dcd09663c7b08474fc1c1b5/C, priority=12, startTime=1733740848629; duration=0sec 2024-12-09T10:40:48,902 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:40:48,902 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1d9d6bba7dcd09663c7b08474fc1c1b5:C 2024-12-09T10:40:48,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741923_1099 (size=12301) 2024-12-09T10:40:48,911 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=667 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/C/3f82dbf5e17c42ef85929e8237c0bb7e 2024-12-09T10:40:48,923 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/A/58b13ded44db4ccd8d78d4abdb8eac6b as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/58b13ded44db4ccd8d78d4abdb8eac6b 2024-12-09T10:40:48,930 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/58b13ded44db4ccd8d78d4abdb8eac6b, entries=150, sequenceid=667, filesize=12.0 K 2024-12-09T10:40:48,933 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/B/5d32ca380ef443d8a047eeb675eab8e3 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/5d32ca380ef443d8a047eeb675eab8e3 2024-12-09T10:40:48,946 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/5d32ca380ef443d8a047eeb675eab8e3, entries=150, sequenceid=667, filesize=12.0 K 2024-12-09T10:40:48,951 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/C/3f82dbf5e17c42ef85929e8237c0bb7e as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/3f82dbf5e17c42ef85929e8237c0bb7e 2024-12-09T10:40:48,957 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/3f82dbf5e17c42ef85929e8237c0bb7e, entries=150, sequenceid=667, filesize=12.0 K 2024-12-09T10:40:48,958 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~259.13 KB/265344, currentSize=93.93 KB/96180 for 1d9d6bba7dcd09663c7b08474fc1c1b5 in 329ms, sequenceid=667, compaction requested=false 2024-12-09T10:40:48,958 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1d9d6bba7dcd09663c7b08474fc1c1b5: 2024-12-09T10:40:49,034 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-09T10:40:49,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:49,037 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing 1d9d6bba7dcd09663c7b08474fc1c1b5 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-09T10:40:49,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(8855): Flush requested on 1d9d6bba7dcd09663c7b08474fc1c1b5 2024-12-09T10:40:49,037 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. as already flushing 2024-12-09T10:40:49,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d9d6bba7dcd09663c7b08474fc1c1b5, store=A 2024-12-09T10:40:49,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:40:49,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d9d6bba7dcd09663c7b08474fc1c1b5, store=B 2024-12-09T10:40:49,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:40:49,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d9d6bba7dcd09663c7b08474fc1c1b5, store=C 2024-12-09T10:40:49,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:40:49,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/A/ce13dd6b12f74d20b2daca43df4831a6 is 50, key is test_row_0/A:col10/1733740848702/Put/seqid=0 2024-12-09T10:40:49,073 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:49,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46398 deadline: 1733740909061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:49,074 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:49,074 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:49,076 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:49,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46390 deadline: 1733740909061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:49,076 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:49,077 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:49,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46362 deadline: 1733740909062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:49,077 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:49,077 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:49,078 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:49,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:49,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46396 deadline: 1733740909063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:49,079 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:49,079 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:49,079 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:49,079 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:49,079 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:49,079 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:49,083 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:49,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46374 deadline: 1733740909073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:49,085 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:49,085 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:49,085 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:49,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741924_1100 (size=12301) 2024-12-09T10:40:49,167 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/B/a076f9c734dc4963995bff79b107ce62 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/a076f9c734dc4963995bff79b107ce62 2024-12-09T10:40:49,182 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1d9d6bba7dcd09663c7b08474fc1c1b5/B of 1d9d6bba7dcd09663c7b08474fc1c1b5 into a076f9c734dc4963995bff79b107ce62(size=12.9 K), total size for store is 24.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:40:49,182 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1d9d6bba7dcd09663c7b08474fc1c1b5: 2024-12-09T10:40:49,182 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., storeName=1d9d6bba7dcd09663c7b08474fc1c1b5/B, priority=12, startTime=1733740848629; duration=0sec 2024-12-09T10:40:49,183 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:40:49,182 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:49,183 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1d9d6bba7dcd09663c7b08474fc1c1b5:B 2024-12-09T10:40:49,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46398 deadline: 1733740909179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:49,183 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:49,183 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:49,183 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:49,183 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:49,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46362 deadline: 1733740909179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:49,184 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:49,184 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:49,184 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:49,191 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:49,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46374 deadline: 1733740909190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:49,192 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:49,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46396 deadline: 1733740909190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:49,192 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:49,193 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:49,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46390 deadline: 1733740909192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:49,193 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:49,193 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:49,193 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:49,193 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:49,193 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:49,193 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:49,193 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:49,193 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:49,398 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:49,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46398 deadline: 1733740909389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:49,399 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:49,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46362 deadline: 1733740909389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:49,400 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:49,400 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:49,400 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:49,400 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:49,400 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:49,400 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:49,401 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:49,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46390 deadline: 1733740909400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:49,404 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:49,404 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:49,404 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:49,404 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:49,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46374 deadline: 1733740909401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:49,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:49,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46396 deadline: 1733740909401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:49,405 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:49,405 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:49,405 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:49,405 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:49,405 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:49,405 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:49,507 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=688 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/A/ce13dd6b12f74d20b2daca43df4831a6 2024-12-09T10:40:49,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/B/f339c8fba74846d097e6952eaaf3a1b1 is 50, key is test_row_0/B:col10/1733740848702/Put/seqid=0 2024-12-09T10:40:49,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741925_1101 (size=12301) 2024-12-09T10:40:49,540 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=688 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/B/f339c8fba74846d097e6952eaaf3a1b1 2024-12-09T10:40:49,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/C/7f4dcacb7db84579bc1204d1b0db656a is 50, key is test_row_0/C:col10/1733740848702/Put/seqid=0 2024-12-09T10:40:49,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741926_1102 (size=12301) 2024-12-09T10:40:49,710 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:49,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46398 deadline: 1733740909709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:49,711 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:49,711 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:49,711 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:49,711 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:49,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46374 deadline: 1733740909709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:49,712 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:49,712 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:49,712 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:49,712 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:49,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46362 deadline: 1733740909710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:49,712 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:49,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46390 deadline: 1733740909710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:49,713 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:49,713 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:49,713 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:49,713 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:49,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46396 deadline: 1733740909710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:49,714 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:49,714 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:49,714 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:49,718 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:49,718 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:49,718 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:49,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-09T10:40:49,981 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=688 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/C/7f4dcacb7db84579bc1204d1b0db656a 2024-12-09T10:40:49,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/A/ce13dd6b12f74d20b2daca43df4831a6 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/ce13dd6b12f74d20b2daca43df4831a6 2024-12-09T10:40:50,011 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/ce13dd6b12f74d20b2daca43df4831a6, entries=150, sequenceid=688, filesize=12.0 K 2024-12-09T10:40:50,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/B/f339c8fba74846d097e6952eaaf3a1b1 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/f339c8fba74846d097e6952eaaf3a1b1 2024-12-09T10:40:50,024 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/f339c8fba74846d097e6952eaaf3a1b1, entries=150, sequenceid=688, filesize=12.0 K 2024-12-09T10:40:50,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/C/7f4dcacb7db84579bc1204d1b0db656a as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/7f4dcacb7db84579bc1204d1b0db656a 2024-12-09T10:40:50,058 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/7f4dcacb7db84579bc1204d1b0db656a, entries=150, sequenceid=688, filesize=12.0 K 2024-12-09T10:40:50,061 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 1d9d6bba7dcd09663c7b08474fc1c1b5 in 1024ms, sequenceid=688, compaction requested=true 2024-12-09T10:40:50,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for 1d9d6bba7dcd09663c7b08474fc1c1b5: 2024-12-09T10:40:50,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:50,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-12-09T10:40:50,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-12-09T10:40:50,073 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 2.3300 sec 2024-12-09T10:40:50,073 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-12-09T10:40:50,073 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3180 sec 2024-12-09T10:40:50,226 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1d9d6bba7dcd09663c7b08474fc1c1b5 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-09T10:40:50,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(8855): Flush requested on 1d9d6bba7dcd09663c7b08474fc1c1b5 2024-12-09T10:40:50,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d9d6bba7dcd09663c7b08474fc1c1b5, store=A 2024-12-09T10:40:50,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:40:50,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d9d6bba7dcd09663c7b08474fc1c1b5, store=B 2024-12-09T10:40:50,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:40:50,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d9d6bba7dcd09663c7b08474fc1c1b5, store=C 2024-12-09T10:40:50,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:40:50,248 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/A/d74757f1901e4618aaa1cb290c8e2f95 is 50, key is test_row_0/A:col10/1733740850224/Put/seqid=0 2024-12-09T10:40:50,300 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:50,300 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:50,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46374 deadline: 1733740910282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:50,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46396 deadline: 1733740910285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:50,301 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:50,301 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:50,301 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:50,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:50,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:50,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:50,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741927_1103 (size=14741) 2024-12-09T10:40:50,309 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:50,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46390 deadline: 1733740910301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:50,311 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:50,311 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:50,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46398 deadline: 1733740910301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:50,311 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:50,311 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:50,312 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:50,312 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:50,312 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:50,313 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:50,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46362 deadline: 1733740910305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:50,314 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:50,314 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:50,314 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:50,414 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:50,414 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:50,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46374 deadline: 1733740910410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:50,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46396 deadline: 1733740910410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:50,415 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:50,415 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:50,415 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:50,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:50,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:50,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:50,420 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:50,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46398 deadline: 1733740910419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:50,420 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:50,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46362 deadline: 1733740910420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:50,421 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:50,421 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:50,421 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:50,421 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:50,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46390 deadline: 1733740910420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:50,422 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:50,422 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:50,422 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:50,423 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:50,423 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:50,423 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:50,619 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:50,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46374 deadline: 1733740910619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:50,620 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:50,620 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:50,620 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:50,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:50,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46396 deadline: 1733740910629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:50,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:50,631 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:50,631 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:50,631 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:50,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46362 deadline: 1733740910630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:50,632 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:50,632 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:50,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46390 deadline: 1733740910630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:50,632 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:50,632 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:50,632 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:50,632 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:50,632 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:50,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46398 deadline: 1733740910630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:50,632 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:50,633 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:50,633 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:50,633 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:50,704 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=704 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/A/d74757f1901e4618aaa1cb290c8e2f95 2024-12-09T10:40:50,728 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/B/295a6c6a0cba4917b0d3eb84047b6888 is 50, key is test_row_0/B:col10/1733740850224/Put/seqid=0 2024-12-09T10:40:50,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741928_1104 (size=12301) 2024-12-09T10:40:50,737 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=704 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/B/295a6c6a0cba4917b0d3eb84047b6888 2024-12-09T10:40:50,746 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/C/066d197e9720486a95743c60e5f0c633 is 50, key is test_row_0/C:col10/1733740850224/Put/seqid=0 2024-12-09T10:40:50,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741929_1105 (size=12301) 2024-12-09T10:40:50,758 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=704 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/C/066d197e9720486a95743c60e5f0c633 2024-12-09T10:40:50,768 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/A/d74757f1901e4618aaa1cb290c8e2f95 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/d74757f1901e4618aaa1cb290c8e2f95 2024-12-09T10:40:50,775 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/d74757f1901e4618aaa1cb290c8e2f95, entries=200, sequenceid=704, filesize=14.4 K 2024-12-09T10:40:50,776 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/B/295a6c6a0cba4917b0d3eb84047b6888 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/295a6c6a0cba4917b0d3eb84047b6888 2024-12-09T10:40:50,785 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/295a6c6a0cba4917b0d3eb84047b6888, entries=150, sequenceid=704, filesize=12.0 K 2024-12-09T10:40:50,791 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/C/066d197e9720486a95743c60e5f0c633 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/066d197e9720486a95743c60e5f0c633 2024-12-09T10:40:50,802 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/066d197e9720486a95743c60e5f0c633, entries=150, sequenceid=704, filesize=12.0 K 2024-12-09T10:40:50,803 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=107.34 KB/109920 for 1d9d6bba7dcd09663c7b08474fc1c1b5 in 577ms, sequenceid=704, compaction requested=true 2024-12-09T10:40:50,804 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1d9d6bba7dcd09663c7b08474fc1c1b5: 2024-12-09T10:40:50,804 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T10:40:50,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1d9d6bba7dcd09663c7b08474fc1c1b5:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T10:40:50,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:40:50,804 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T10:40:50,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1d9d6bba7dcd09663c7b08474fc1c1b5:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T10:40:50,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:40:50,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1d9d6bba7dcd09663c7b08474fc1c1b5:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T10:40:50,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T10:40:50,806 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52564 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T10:40:50,806 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1541): 1d9d6bba7dcd09663c7b08474fc1c1b5/A is initiating minor compaction (all files) 2024-12-09T10:40:50,806 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1d9d6bba7dcd09663c7b08474fc1c1b5/A in TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:50,806 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50124 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T10:40:50,807 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/18757bffe85a4749931bd1bdf4978283, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/58b13ded44db4ccd8d78d4abdb8eac6b, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/ce13dd6b12f74d20b2daca43df4831a6, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/d74757f1901e4618aaa1cb290c8e2f95] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp, totalSize=51.3 K 2024-12-09T10:40:50,807 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1541): 1d9d6bba7dcd09663c7b08474fc1c1b5/B is initiating minor compaction (all files) 2024-12-09T10:40:50,807 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1d9d6bba7dcd09663c7b08474fc1c1b5/B in TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:50,807 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/a076f9c734dc4963995bff79b107ce62, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/5d32ca380ef443d8a047eeb675eab8e3, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/f339c8fba74846d097e6952eaaf3a1b1, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/295a6c6a0cba4917b0d3eb84047b6888] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp, totalSize=48.9 K 2024-12-09T10:40:50,807 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 18757bffe85a4749931bd1bdf4978283, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=414, earliestPutTs=1733740847379 2024-12-09T10:40:50,807 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting a076f9c734dc4963995bff79b107ce62, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=414, earliestPutTs=1733740847379 2024-12-09T10:40:50,808 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 58b13ded44db4ccd8d78d4abdb8eac6b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=667, earliestPutTs=1733740848588 2024-12-09T10:40:50,808 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d32ca380ef443d8a047eeb675eab8e3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=667, earliestPutTs=1733740848588 2024-12-09T10:40:50,809 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting ce13dd6b12f74d20b2daca43df4831a6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=688, earliestPutTs=1733740848702 2024-12-09T10:40:50,809 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting f339c8fba74846d097e6952eaaf3a1b1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=688, earliestPutTs=1733740848702 2024-12-09T10:40:50,809 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting d74757f1901e4618aaa1cb290c8e2f95, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=704, earliestPutTs=1733740849050 2024-12-09T10:40:50,809 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 295a6c6a0cba4917b0d3eb84047b6888, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=704, earliestPutTs=1733740849061 2024-12-09T10:40:50,828 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1d9d6bba7dcd09663c7b08474fc1c1b5#A#compaction#93 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:40:50,830 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/A/22c76c9df31d4d3ba406386ffb7fbcf8 is 50, key is test_row_0/A:col10/1733740850224/Put/seqid=0 2024-12-09T10:40:50,832 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1d9d6bba7dcd09663c7b08474fc1c1b5#B#compaction#94 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:40:50,834 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/B/787e78687c7047eb8fc53206bbb6f359 is 50, key is test_row_0/B:col10/1733740850224/Put/seqid=0 2024-12-09T10:40:50,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741930_1106 (size=13357) 2024-12-09T10:40:50,860 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/A/22c76c9df31d4d3ba406386ffb7fbcf8 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/22c76c9df31d4d3ba406386ffb7fbcf8 2024-12-09T10:40:50,870 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1d9d6bba7dcd09663c7b08474fc1c1b5/A of 1d9d6bba7dcd09663c7b08474fc1c1b5 into 22c76c9df31d4d3ba406386ffb7fbcf8(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:40:50,870 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1d9d6bba7dcd09663c7b08474fc1c1b5: 2024-12-09T10:40:50,870 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., storeName=1d9d6bba7dcd09663c7b08474fc1c1b5/A, priority=12, startTime=1733740850804; duration=0sec 2024-12-09T10:40:50,870 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T10:40:50,870 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1d9d6bba7dcd09663c7b08474fc1c1b5:A 2024-12-09T10:40:50,871 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T10:40:50,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741931_1107 (size=13357) 2024-12-09T10:40:50,874 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50124 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T10:40:50,874 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1541): 1d9d6bba7dcd09663c7b08474fc1c1b5/C is initiating minor compaction (all files) 2024-12-09T10:40:50,874 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1d9d6bba7dcd09663c7b08474fc1c1b5/C in TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:50,875 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/7c2d9230f5d04252b1c75918152a42ea, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/3f82dbf5e17c42ef85929e8237c0bb7e, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/7f4dcacb7db84579bc1204d1b0db656a, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/066d197e9720486a95743c60e5f0c633] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp, totalSize=48.9 K 2024-12-09T10:40:50,875 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7c2d9230f5d04252b1c75918152a42ea, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=414, earliestPutTs=1733740847379 2024-12-09T10:40:50,876 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3f82dbf5e17c42ef85929e8237c0bb7e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=667, earliestPutTs=1733740848588 2024-12-09T10:40:50,876 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f4dcacb7db84579bc1204d1b0db656a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=688, earliestPutTs=1733740848702 2024-12-09T10:40:50,877 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 066d197e9720486a95743c60e5f0c633, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=704, earliestPutTs=1733740849061 2024-12-09T10:40:50,887 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/B/787e78687c7047eb8fc53206bbb6f359 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/787e78687c7047eb8fc53206bbb6f359 2024-12-09T10:40:50,898 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1d9d6bba7dcd09663c7b08474fc1c1b5/B of 1d9d6bba7dcd09663c7b08474fc1c1b5 into 787e78687c7047eb8fc53206bbb6f359(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:40:50,898 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1d9d6bba7dcd09663c7b08474fc1c1b5: 2024-12-09T10:40:50,898 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., storeName=1d9d6bba7dcd09663c7b08474fc1c1b5/B, priority=12, startTime=1733740850804; duration=0sec 2024-12-09T10:40:50,898 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:40:50,898 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1d9d6bba7dcd09663c7b08474fc1c1b5:B 2024-12-09T10:40:50,902 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1d9d6bba7dcd09663c7b08474fc1c1b5#C#compaction#95 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:40:50,902 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/C/9e2e3b4881834b5f96ecac77e3a5b9aa is 50, key is test_row_0/C:col10/1733740850224/Put/seqid=0 2024-12-09T10:40:50,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741932_1108 (size=13357) 2024-12-09T10:40:50,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(8855): Flush requested on 1d9d6bba7dcd09663c7b08474fc1c1b5 2024-12-09T10:40:50,933 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1d9d6bba7dcd09663c7b08474fc1c1b5 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-09T10:40:50,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d9d6bba7dcd09663c7b08474fc1c1b5, store=A 2024-12-09T10:40:50,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:40:50,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d9d6bba7dcd09663c7b08474fc1c1b5, store=B 2024-12-09T10:40:50,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:40:50,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d9d6bba7dcd09663c7b08474fc1c1b5, store=C 2024-12-09T10:40:50,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:40:50,944 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/A/eef7d35779bb4034a746303a4d51ff63 is 50, key is test_row_0/A:col10/1733740850281/Put/seqid=0 2024-12-09T10:40:50,945 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/C/9e2e3b4881834b5f96ecac77e3a5b9aa as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/9e2e3b4881834b5f96ecac77e3a5b9aa 2024-12-09T10:40:50,959 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1d9d6bba7dcd09663c7b08474fc1c1b5/C of 1d9d6bba7dcd09663c7b08474fc1c1b5 into 9e2e3b4881834b5f96ecac77e3a5b9aa(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:40:50,959 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1d9d6bba7dcd09663c7b08474fc1c1b5: 2024-12-09T10:40:50,959 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., storeName=1d9d6bba7dcd09663c7b08474fc1c1b5/C, priority=12, startTime=1733740850804; duration=0sec 2024-12-09T10:40:50,959 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:40:50,959 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1d9d6bba7dcd09663c7b08474fc1c1b5:C 2024-12-09T10:40:50,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:50,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46362 deadline: 1733740910950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:50,960 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:50,960 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:50,960 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:50,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:50,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46398 deadline: 1733740910951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:50,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:50,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:50,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:50,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741933_1109 (size=14741) 2024-12-09T10:40:50,965 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=726 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/A/eef7d35779bb4034a746303a4d51ff63 2024-12-09T10:40:50,969 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:50,969 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:50,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46374 deadline: 1733740910957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:50,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 217 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46396 deadline: 1733740910959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:50,969 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:50,969 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:50,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46390 deadline: 1733740910960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:50,969 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:50,969 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:50,969 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:50,969 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:50,970 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:50,970 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:50,970 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:50,970 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:50,986 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/B/a234aa97e06b46a3b5c0c52f0af06c4a is 50, key is test_row_0/B:col10/1733740850281/Put/seqid=0 2024-12-09T10:40:51,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741934_1110 (size=12301) 2024-12-09T10:40:51,011 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=726 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/B/a234aa97e06b46a3b5c0c52f0af06c4a 2024-12-09T10:40:51,030 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/C/ba211b5061e240d8bac6272d470712db is 50, key is test_row_0/C:col10/1733740850281/Put/seqid=0 2024-12-09T10:40:51,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741935_1111 (size=12301) 2024-12-09T10:40:51,045 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=726 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/C/ba211b5061e240d8bac6272d470712db 2024-12-09T10:40:51,053 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/A/eef7d35779bb4034a746303a4d51ff63 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/eef7d35779bb4034a746303a4d51ff63 2024-12-09T10:40:51,060 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/eef7d35779bb4034a746303a4d51ff63, entries=200, sequenceid=726, filesize=14.4 K 2024-12-09T10:40:51,064 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/B/a234aa97e06b46a3b5c0c52f0af06c4a as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/a234aa97e06b46a3b5c0c52f0af06c4a 2024-12-09T10:40:51,071 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/a234aa97e06b46a3b5c0c52f0af06c4a, entries=150, sequenceid=726, filesize=12.0 K 2024-12-09T10:40:51,072 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/C/ba211b5061e240d8bac6272d470712db as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/ba211b5061e240d8bac6272d470712db 2024-12-09T10:40:51,080 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/ba211b5061e240d8bac6272d470712db, entries=150, sequenceid=726, filesize=12.0 K 2024-12-09T10:40:51,081 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:51,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46362 deadline: 1733740911070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:51,083 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:51,083 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:51,084 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 1d9d6bba7dcd09663c7b08474fc1c1b5 in 151ms, sequenceid=726, compaction requested=false 2024-12-09T10:40:51,084 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:51,084 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1d9d6bba7dcd09663c7b08474fc1c1b5: 2024-12-09T10:40:51,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(8855): Flush requested on 1d9d6bba7dcd09663c7b08474fc1c1b5 2024-12-09T10:40:51,085 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1d9d6bba7dcd09663c7b08474fc1c1b5 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-09T10:40:51,085 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d9d6bba7dcd09663c7b08474fc1c1b5, store=A 2024-12-09T10:40:51,085 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:40:51,085 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d9d6bba7dcd09663c7b08474fc1c1b5, store=B 2024-12-09T10:40:51,085 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:40:51,085 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d9d6bba7dcd09663c7b08474fc1c1b5, store=C 2024-12-09T10:40:51,085 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:40:51,100 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/A/c63debb455ec4fe9865686c5d57123af is 50, key is test_row_0/A:col10/1733740850955/Put/seqid=0 2024-12-09T10:40:51,157 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:51,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46390 deadline: 1733740911144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:51,159 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:51,159 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:51,159 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:51,159 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:51,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 222 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46396 deadline: 1733740911144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:51,160 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:51,160 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:51,160 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:51,160 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:51,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46374 deadline: 1733740911145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:51,161 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:51,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 221 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46398 deadline: 1733740911146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:51,162 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:51,162 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:51,162 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:51,162 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:51,162 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:51,162 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:51,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741936_1112 (size=14741) 2024-12-09T10:40:51,165 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=740 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/A/c63debb455ec4fe9865686c5d57123af 2024-12-09T10:40:51,184 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/B/23545d3781dc4152834aeeb232077612 is 50, key is test_row_0/B:col10/1733740850955/Put/seqid=0 2024-12-09T10:40:51,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741937_1113 (size=12301) 2024-12-09T10:40:51,274 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:51,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46390 deadline: 1733740911270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:51,276 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:51,276 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:51,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 222 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46398 deadline: 1733740911270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:51,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46374 deadline: 1733740911270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:51,277 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:51,277 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:51,277 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:51,277 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:51,277 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:51,277 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:51,275 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:51,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 223 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46396 deadline: 1733740911270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:51,278 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:51,278 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:51,278 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:51,279 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:51,280 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:51,280 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:51,293 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:51,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46362 deadline: 1733740911290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:51,295 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:51,296 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:51,296 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:51,494 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:51,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 223 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46398 deadline: 1733740911490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:51,495 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:51,495 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:51,496 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:51,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46374 deadline: 1733740911490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:51,496 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:51,496 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:51,496 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:51,496 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:51,496 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:51,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46390 deadline: 1733740911490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:51,496 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:51,497 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:51,497 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:51,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 224 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46396 deadline: 1733740911490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:51,497 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:51,497 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:51,497 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:51,497 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:51,601 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=740 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/B/23545d3781dc4152834aeeb232077612 2024-12-09T10:40:51,603 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:51,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46362 deadline: 1733740911600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:51,604 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:51,604 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:51,604 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:51,618 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/C/2d477cd8ac9f4d498dae7a8f038d3d07 is 50, key is test_row_0/C:col10/1733740850955/Put/seqid=0 2024-12-09T10:40:51,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741938_1114 (size=12301) 2024-12-09T10:40:51,801 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:51,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 224 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46398 deadline: 1733740911799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:51,802 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:51,803 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:51,803 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:51,803 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:51,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 217 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46374 deadline: 1733740911800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:51,803 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:51,804 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:51,804 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:51,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46390 deadline: 1733740911800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:51,804 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:51,804 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:51,804 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:51,804 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:51,811 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:51,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 225 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46396 deadline: 1733740911810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:51,812 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:51,812 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:51,812 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:51,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-09T10:40:51,910 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T10:40:51,911 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-09T10:40:51,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-12-09T10:40:51,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-09T10:40:51,914 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T10:40:51,915 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T10:40:51,915 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T10:40:52,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-09T10:40:52,068 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-09T10:40:52,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:52,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. as already flushing 2024-12-09T10:40:52,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:52,069 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:40:52,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:40:52,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:40:52,071 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=740 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/C/2d477cd8ac9f4d498dae7a8f038d3d07 2024-12-09T10:40:52,078 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/A/c63debb455ec4fe9865686c5d57123af as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/c63debb455ec4fe9865686c5d57123af 2024-12-09T10:40:52,083 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/c63debb455ec4fe9865686c5d57123af, entries=200, sequenceid=740, filesize=14.4 K 2024-12-09T10:40:52,084 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/B/23545d3781dc4152834aeeb232077612 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/23545d3781dc4152834aeeb232077612 2024-12-09T10:40:52,090 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/23545d3781dc4152834aeeb232077612, entries=150, sequenceid=740, filesize=12.0 K 2024-12-09T10:40:52,091 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/C/2d477cd8ac9f4d498dae7a8f038d3d07 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/2d477cd8ac9f4d498dae7a8f038d3d07 2024-12-09T10:40:52,103 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/2d477cd8ac9f4d498dae7a8f038d3d07, entries=150, sequenceid=740, filesize=12.0 K 2024-12-09T10:40:52,104 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 1d9d6bba7dcd09663c7b08474fc1c1b5 in 1019ms, sequenceid=740, compaction requested=true 2024-12-09T10:40:52,104 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1d9d6bba7dcd09663c7b08474fc1c1b5: 2024-12-09T10:40:52,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1d9d6bba7dcd09663c7b08474fc1c1b5:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T10:40:52,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:40:52,104 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T10:40:52,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1d9d6bba7dcd09663c7b08474fc1c1b5:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T10:40:52,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:40:52,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1d9d6bba7dcd09663c7b08474fc1c1b5:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T10:40:52,104 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T10:40:52,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T10:40:52,105 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37959 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T10:40:52,105 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42839 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T10:40:52,106 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1541): 1d9d6bba7dcd09663c7b08474fc1c1b5/A is initiating minor compaction (all files) 2024-12-09T10:40:52,106 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1541): 1d9d6bba7dcd09663c7b08474fc1c1b5/B is initiating minor compaction (all files) 2024-12-09T10:40:52,106 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1d9d6bba7dcd09663c7b08474fc1c1b5/A in TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:52,106 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1d9d6bba7dcd09663c7b08474fc1c1b5/B in TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:52,106 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/787e78687c7047eb8fc53206bbb6f359, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/a234aa97e06b46a3b5c0c52f0af06c4a, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/23545d3781dc4152834aeeb232077612] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp, totalSize=37.1 K 2024-12-09T10:40:52,106 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/22c76c9df31d4d3ba406386ffb7fbcf8, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/eef7d35779bb4034a746303a4d51ff63, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/c63debb455ec4fe9865686c5d57123af] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp, totalSize=41.8 K 2024-12-09T10:40:52,106 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 787e78687c7047eb8fc53206bbb6f359, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=704, earliestPutTs=1733740849061 2024-12-09T10:40:52,106 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 22c76c9df31d4d3ba406386ffb7fbcf8, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=704, earliestPutTs=1733740849061 2024-12-09T10:40:52,107 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting a234aa97e06b46a3b5c0c52f0af06c4a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=726, earliestPutTs=1733740850281 2024-12-09T10:40:52,107 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting eef7d35779bb4034a746303a4d51ff63, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=726, earliestPutTs=1733740850281 2024-12-09T10:40:52,107 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 23545d3781dc4152834aeeb232077612, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=740, earliestPutTs=1733740850955 2024-12-09T10:40:52,107 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting c63debb455ec4fe9865686c5d57123af, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=740, earliestPutTs=1733740850946 2024-12-09T10:40:52,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(8855): Flush requested on 1d9d6bba7dcd09663c7b08474fc1c1b5 2024-12-09T10:40:52,115 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1d9d6bba7dcd09663c7b08474fc1c1b5 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-09T10:40:52,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d9d6bba7dcd09663c7b08474fc1c1b5, store=A 2024-12-09T10:40:52,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:40:52,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d9d6bba7dcd09663c7b08474fc1c1b5, store=B 2024-12-09T10:40:52,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:40:52,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d9d6bba7dcd09663c7b08474fc1c1b5, store=C 2024-12-09T10:40:52,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:40:52,118 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1d9d6bba7dcd09663c7b08474fc1c1b5#B#compaction#102 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:40:52,119 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/B/d2b6831b24c244ea86451c8b656f9d21 is 50, key is test_row_0/B:col10/1733740850955/Put/seqid=0 2024-12-09T10:40:52,121 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1d9d6bba7dcd09663c7b08474fc1c1b5#A#compaction#103 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:40:52,122 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/A/0dd8a387b31f4cc99fc4500ee1427b57 is 50, key is test_row_0/A:col10/1733740850955/Put/seqid=0 2024-12-09T10:40:52,129 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/A/fb074d5691f54483b0300bc50f829eb2 is 50, key is test_row_0/A:col10/1733740851133/Put/seqid=0 2024-12-09T10:40:52,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741939_1115 (size=13459) 2024-12-09T10:40:52,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741940_1116 (size=13459) 2024-12-09T10:40:52,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741941_1117 (size=14741) 2024-12-09T10:40:52,163 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/B/d2b6831b24c244ea86451c8b656f9d21 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/d2b6831b24c244ea86451c8b656f9d21 2024-12-09T10:40:52,173 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1d9d6bba7dcd09663c7b08474fc1c1b5/B of 1d9d6bba7dcd09663c7b08474fc1c1b5 into d2b6831b24c244ea86451c8b656f9d21(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:40:52,173 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1d9d6bba7dcd09663c7b08474fc1c1b5: 2024-12-09T10:40:52,173 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., storeName=1d9d6bba7dcd09663c7b08474fc1c1b5/B, priority=13, startTime=1733740852104; duration=0sec 2024-12-09T10:40:52,174 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T10:40:52,174 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1d9d6bba7dcd09663c7b08474fc1c1b5:B 2024-12-09T10:40:52,174 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T10:40:52,175 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37959 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T10:40:52,176 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1541): 1d9d6bba7dcd09663c7b08474fc1c1b5/C is initiating minor compaction (all files) 2024-12-09T10:40:52,176 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1d9d6bba7dcd09663c7b08474fc1c1b5/C in TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:52,176 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/9e2e3b4881834b5f96ecac77e3a5b9aa, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/ba211b5061e240d8bac6272d470712db, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/2d477cd8ac9f4d498dae7a8f038d3d07] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp, totalSize=37.1 K 2024-12-09T10:40:52,177 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e2e3b4881834b5f96ecac77e3a5b9aa, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=704, earliestPutTs=1733740849061 2024-12-09T10:40:52,177 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting ba211b5061e240d8bac6272d470712db, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=726, earliestPutTs=1733740850281 2024-12-09T10:40:52,178 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d477cd8ac9f4d498dae7a8f038d3d07, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=740, earliestPutTs=1733740850955 2024-12-09T10:40:52,181 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:52,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46362 deadline: 1733740912177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:52,182 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:52,182 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:52,182 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:52,187 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1d9d6bba7dcd09663c7b08474fc1c1b5#C#compaction#105 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:40:52,188 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/C/24051a12bb5b43a58ab13ad2b5c5ff5a is 50, key is test_row_0/C:col10/1733740850955/Put/seqid=0 2024-12-09T10:40:52,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741942_1118 (size=13459) 2024-12-09T10:40:52,223 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-09T10:40:52,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:52,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. as already flushing 2024-12-09T10:40:52,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:52,223 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:40:52,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:40:52,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:40:52,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-09T10:40:52,295 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:52,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46362 deadline: 1733740912289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:52,296 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:52,296 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:52,296 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:52,310 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:52,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 225 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46398 deadline: 1733740912309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:52,311 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:52,311 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:52,311 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:52,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 218 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46374 deadline: 1733740912309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:52,311 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:52,311 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:52,311 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:52,311 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:52,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46390 deadline: 1733740912309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:52,311 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:52,312 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:52,312 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:52,312 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:52,324 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:52,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 226 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46396 deadline: 1733740912320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:52,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:52,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:52,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:52,376 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-09T10:40:52,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:52,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. as already flushing 2024-12-09T10:40:52,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:52,377 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:40:52,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:40:52,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:40:52,500 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:52,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46362 deadline: 1733740912499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:52,500 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:52,500 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:52,500 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:52,530 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-09T10:40:52,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:52,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. as already flushing 2024-12-09T10:40:52,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:52,530 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:40:52,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:40:52,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:40:52,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-09T10:40:52,557 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=760 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/A/fb074d5691f54483b0300bc50f829eb2 2024-12-09T10:40:52,558 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/A/0dd8a387b31f4cc99fc4500ee1427b57 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/0dd8a387b31f4cc99fc4500ee1427b57 2024-12-09T10:40:52,566 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1d9d6bba7dcd09663c7b08474fc1c1b5/A of 1d9d6bba7dcd09663c7b08474fc1c1b5 into 0dd8a387b31f4cc99fc4500ee1427b57(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:40:52,566 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1d9d6bba7dcd09663c7b08474fc1c1b5: 2024-12-09T10:40:52,566 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., storeName=1d9d6bba7dcd09663c7b08474fc1c1b5/A, priority=13, startTime=1733740852104; duration=0sec 2024-12-09T10:40:52,566 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:40:52,566 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1d9d6bba7dcd09663c7b08474fc1c1b5:A 2024-12-09T10:40:52,567 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/B/ab361d23101f4836bfefde171ddbe50e is 50, key is test_row_0/B:col10/1733740851133/Put/seqid=0 2024-12-09T10:40:52,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741943_1119 (size=12301) 2024-12-09T10:40:52,605 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/C/24051a12bb5b43a58ab13ad2b5c5ff5a as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/24051a12bb5b43a58ab13ad2b5c5ff5a 2024-12-09T10:40:52,611 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1d9d6bba7dcd09663c7b08474fc1c1b5/C of 1d9d6bba7dcd09663c7b08474fc1c1b5 into 24051a12bb5b43a58ab13ad2b5c5ff5a(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:40:52,611 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1d9d6bba7dcd09663c7b08474fc1c1b5: 2024-12-09T10:40:52,611 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., storeName=1d9d6bba7dcd09663c7b08474fc1c1b5/C, priority=13, startTime=1733740852104; duration=0sec 2024-12-09T10:40:52,611 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:40:52,612 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1d9d6bba7dcd09663c7b08474fc1c1b5:C 2024-12-09T10:40:52,683 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-09T10:40:52,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:52,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. as already flushing 2024-12-09T10:40:52,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:52,684 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:40:52,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:40:52,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:40:52,810 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:52,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46362 deadline: 1733740912809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:52,811 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:52,811 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:52,811 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:52,837 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-09T10:40:52,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:52,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. as already flushing 2024-12-09T10:40:52,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:52,837 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:40:52,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:40:52,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:40:52,973 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=760 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/B/ab361d23101f4836bfefde171ddbe50e 2024-12-09T10:40:52,982 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/C/810b4b81640648f7879f8a9fa6675128 is 50, key is test_row_0/C:col10/1733740851133/Put/seqid=0 2024-12-09T10:40:52,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741944_1120 (size=12301) 2024-12-09T10:40:52,990 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-09T10:40:52,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:52,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. as already flushing 2024-12-09T10:40:52,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:52,991 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:40:52,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:40:52,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:40:52,996 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-12-09T10:40:52,997 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T10:40:52,999 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-12-09T10:40:53,000 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T10:40:53,000 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-12-09T10:40:53,000 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T10:40:53,001 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-12-09T10:40:53,001 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T10:40:53,002 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:40:53,002 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:40:53,002 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:40:53,002 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:40:53,003 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:40:53,003 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:40:53,003 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T10:40:53,003 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:40:53,003 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T10:40:53,003 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T10:40:53,003 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:40:53,003 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T10:40:53,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-09T10:40:53,144 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-09T10:40:53,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:53,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. as already flushing 2024-12-09T10:40:53,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:53,144 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:40:53,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:40:53,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:40:53,297 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-09T10:40:53,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:53,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. as already flushing 2024-12-09T10:40:53,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:53,298 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:40:53,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:40:53,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:40:53,320 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:53,320 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:53,320 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:53,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 226 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46398 deadline: 1733740913319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:53,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46362 deadline: 1733740913319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:53,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 219 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46374 deadline: 1733740913319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:53,320 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:53,320 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:53,320 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:53,320 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 7ea43b39fc24,36101,1733740827678 for 'test_row_0' in 1d9d6bba7dcd09663c7b08474fc1c1b5 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T10:40:53,320 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 7ea43b39fc24,36101,1733740827678 for 'test_row_2' in 1d9d6bba7dcd09663c7b08474fc1c1b5 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2174 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T10:40:53,321 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:53,321 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:53,321 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:53,321 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:53,321 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:53,321 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:53,330 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:53,330 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:40:53,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46390 deadline: 1733740913330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:53,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 227 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46396 deadline: 1733740913330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:53,331 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 7ea43b39fc24,36101,1733740827678 for 'test_row_1' in 1d9d6bba7dcd09663c7b08474fc1c1b5 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T10:40:53,331 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 7ea43b39fc24,36101,1733740827678 for 'test_row_1' in 1d9d6bba7dcd09663c7b08474fc1c1b5 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T10:40:53,332 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:53,332 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:53,332 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:53,332 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:40:53,332 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:53,332 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:40:53,389 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=760 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/C/810b4b81640648f7879f8a9fa6675128 2024-12-09T10:40:53,395 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/A/fb074d5691f54483b0300bc50f829eb2 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/fb074d5691f54483b0300bc50f829eb2 2024-12-09T10:40:53,399 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/fb074d5691f54483b0300bc50f829eb2, entries=200, sequenceid=760, filesize=14.4 K 2024-12-09T10:40:53,400 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/B/ab361d23101f4836bfefde171ddbe50e as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/ab361d23101f4836bfefde171ddbe50e 2024-12-09T10:40:53,404 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/ab361d23101f4836bfefde171ddbe50e, entries=150, sequenceid=760, filesize=12.0 K 2024-12-09T10:40:53,405 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/C/810b4b81640648f7879f8a9fa6675128 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/810b4b81640648f7879f8a9fa6675128 2024-12-09T10:40:53,409 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/810b4b81640648f7879f8a9fa6675128, entries=150, sequenceid=760, filesize=12.0 K 2024-12-09T10:40:53,410 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 1d9d6bba7dcd09663c7b08474fc1c1b5 in 1295ms, sequenceid=760, compaction requested=false 2024-12-09T10:40:53,410 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1d9d6bba7dcd09663c7b08474fc1c1b5: 2024-12-09T10:40:53,451 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-09T10:40:53,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:53,451 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing 1d9d6bba7dcd09663c7b08474fc1c1b5 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-09T10:40:53,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d9d6bba7dcd09663c7b08474fc1c1b5, store=A 2024-12-09T10:40:53,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:40:53,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d9d6bba7dcd09663c7b08474fc1c1b5, store=B 2024-12-09T10:40:53,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:40:53,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d9d6bba7dcd09663c7b08474fc1c1b5, store=C 2024-12-09T10:40:53,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:40:53,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/A/ab85214b0d864922a32052afbf486d34 is 50, key is test_row_0/A:col10/1733740852155/Put/seqid=0 2024-12-09T10:40:53,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741945_1121 (size=12301) 2024-12-09T10:40:53,860 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=775 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/A/ab85214b0d864922a32052afbf486d34 2024-12-09T10:40:53,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/B/879dfbc72d6343729e1637029b31f4f6 is 50, key is test_row_0/B:col10/1733740852155/Put/seqid=0 2024-12-09T10:40:53,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741946_1122 (size=12301) 2024-12-09T10:40:54,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-09T10:40:54,273 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=775 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/B/879dfbc72d6343729e1637029b31f4f6 2024-12-09T10:40:54,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/C/7206cddfadc448d098ced9b420b7aedb is 50, key is test_row_0/C:col10/1733740852155/Put/seqid=0 2024-12-09T10:40:54,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741947_1123 (size=12301) 2024-12-09T10:40:54,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(8855): Flush requested on 1d9d6bba7dcd09663c7b08474fc1c1b5 2024-12-09T10:40:54,331 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. as already flushing 2024-12-09T10:40:54,331 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-12-09T10:40:54,331 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T10:40:54,331 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:40:54,331 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:40:54,331 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T10:40:54,685 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=775 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/C/7206cddfadc448d098ced9b420b7aedb 2024-12-09T10:40:54,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/A/ab85214b0d864922a32052afbf486d34 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/ab85214b0d864922a32052afbf486d34 2024-12-09T10:40:54,694 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/ab85214b0d864922a32052afbf486d34, entries=150, sequenceid=775, filesize=12.0 K 2024-12-09T10:40:54,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/B/879dfbc72d6343729e1637029b31f4f6 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/879dfbc72d6343729e1637029b31f4f6 2024-12-09T10:40:54,699 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/879dfbc72d6343729e1637029b31f4f6, entries=150, sequenceid=775, filesize=12.0 K 2024-12-09T10:40:54,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/C/7206cddfadc448d098ced9b420b7aedb as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/7206cddfadc448d098ced9b420b7aedb 2024-12-09T10:40:54,704 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/7206cddfadc448d098ced9b420b7aedb, entries=150, sequenceid=775, filesize=12.0 K 2024-12-09T10:40:54,705 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=6.71 KB/6870 for 1d9d6bba7dcd09663c7b08474fc1c1b5 in 1254ms, sequenceid=775, compaction requested=true 2024-12-09T10:40:54,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for 1d9d6bba7dcd09663c7b08474fc1c1b5: 2024-12-09T10:40:54,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:54,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-12-09T10:40:54,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-12-09T10:40:54,708 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-12-09T10:40:54,709 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7920 sec 2024-12-09T10:40:54,710 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 2.7980 sec 2024-12-09T10:40:55,342 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-12-09T10:40:55,342 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-12-09T10:40:55,342 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-12-09T10:40:55,342 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T10:40:55,342 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T10:40:55,342 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T10:40:55,342 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:40:55,342 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:40:55,342 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:40:55,342 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:40:55,342 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:40:55,342 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:40:55,343 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T10:40:55,343 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T10:40:55,343 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T10:40:55,352 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-12-09T10:40:55,352 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T10:40:55,352 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:40:55,352 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:40:55,352 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T10:40:55,933 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-09T10:40:56,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-09T10:40:56,080 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T10:40:56,080 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-09T10:40:56,080 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 97 2024-12-09T10:40:56,080 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 166 2024-12-09T10:40:56,080 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 166 2024-12-09T10:40:56,080 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 81 2024-12-09T10:40:56,080 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 158 2024-12-09T10:40:56,080 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-09T10:40:56,080 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2554 2024-12-09T10:40:56,080 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2541 2024-12-09T10:40:56,080 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-09T10:40:56,080 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1167 2024-12-09T10:40:56,080 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3497 rows 2024-12-09T10:40:56,080 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1157 2024-12-09T10:40:56,080 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3470 rows 2024-12-09T10:40:56,080 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-09T10:40:56,081 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T10:40:56,081 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:40:56,081 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:40:56,081 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T10:40:56,085 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-12-09T10:40:56,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-09T10:40:56,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-09T10:40:56,092 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733740856092"}]},"ts":"1733740856092"} 2024-12-09T10:40:56,094 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-09T10:40:56,094 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-09T10:40:56,096 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-09T10:40:56,099 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1d9d6bba7dcd09663c7b08474fc1c1b5, UNASSIGN}] 2024-12-09T10:40:56,100 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1d9d6bba7dcd09663c7b08474fc1c1b5, UNASSIGN 2024-12-09T10:40:56,101 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=1d9d6bba7dcd09663c7b08474fc1c1b5, regionState=CLOSING, regionLocation=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:56,103 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1d9d6bba7dcd09663c7b08474fc1c1b5, UNASSIGN because future has completed 2024-12-09T10:40:56,103 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-09T10:40:56,104 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678}] 2024-12-09T10:40:56,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-09T10:40:56,260 INFO [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(122): Close 1d9d6bba7dcd09663c7b08474fc1c1b5 2024-12-09T10:40:56,260 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-09T10:40:56,261 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1722): Closing 1d9d6bba7dcd09663c7b08474fc1c1b5, disabling compactions & flushes 2024-12-09T10:40:56,261 INFO [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:56,261 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:56,261 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. after waiting 0 ms 2024-12-09T10:40:56,261 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:56,261 INFO [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(2902): Flushing 1d9d6bba7dcd09663c7b08474fc1c1b5 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-09T10:40:56,262 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d9d6bba7dcd09663c7b08474fc1c1b5, store=A 2024-12-09T10:40:56,262 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:40:56,262 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d9d6bba7dcd09663c7b08474fc1c1b5, store=B 2024-12-09T10:40:56,262 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:40:56,262 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d9d6bba7dcd09663c7b08474fc1c1b5, store=C 2024-12-09T10:40:56,262 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:40:56,266 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/A/434156d5247e4194ab7e10504139314a is 50, key is test_row_0/A:col10/1733740855340/Put/seqid=0 2024-12-09T10:40:56,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741948_1124 (size=12301) 2024-12-09T10:40:56,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-09T10:40:56,671 INFO [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=783 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/A/434156d5247e4194ab7e10504139314a 2024-12-09T10:40:56,679 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/B/0b9d4f28369f45da9a384403dbc5e3f3 is 50, key is test_row_0/B:col10/1733740855340/Put/seqid=0 2024-12-09T10:40:56,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741949_1125 (size=12301) 2024-12-09T10:40:56,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-09T10:40:57,084 INFO [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=783 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/B/0b9d4f28369f45da9a384403dbc5e3f3 2024-12-09T10:40:57,091 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/C/2c34d63b62c44f9e90a0533b12e6cf92 is 50, key is test_row_0/C:col10/1733740855340/Put/seqid=0 2024-12-09T10:40:57,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741950_1126 (size=12301) 2024-12-09T10:40:57,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-09T10:40:57,496 INFO [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=783 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/C/2c34d63b62c44f9e90a0533b12e6cf92 2024-12-09T10:40:57,502 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/A/434156d5247e4194ab7e10504139314a as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/434156d5247e4194ab7e10504139314a 2024-12-09T10:40:57,506 INFO [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/434156d5247e4194ab7e10504139314a, entries=150, sequenceid=783, filesize=12.0 K 2024-12-09T10:40:57,507 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/B/0b9d4f28369f45da9a384403dbc5e3f3 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/0b9d4f28369f45da9a384403dbc5e3f3 2024-12-09T10:40:57,511 INFO [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/0b9d4f28369f45da9a384403dbc5e3f3, entries=150, sequenceid=783, filesize=12.0 K 2024-12-09T10:40:57,512 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/.tmp/C/2c34d63b62c44f9e90a0533b12e6cf92 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/2c34d63b62c44f9e90a0533b12e6cf92 2024-12-09T10:40:57,516 INFO [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/2c34d63b62c44f9e90a0533b12e6cf92, entries=150, sequenceid=783, filesize=12.0 K 2024-12-09T10:40:57,517 INFO [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 1d9d6bba7dcd09663c7b08474fc1c1b5 in 1256ms, sequenceid=783, compaction requested=true 2024-12-09T10:40:57,517 DEBUG [StoreCloser-TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/0cf007a7125d4686b4d48808b19fa98b, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/a9ed01b2d09446af94cac32ee8f8212e, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/4b4ce92d705b4485b69a61cf69095a0f, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/32fe83a127de402ea134cac0a94c1b3e, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/996018c21d024fffb1aa2bc5adb98387, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/fd326f8fa2024083ab10771dc379cacd, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/519baef860a141f2b3ec677210dad12d, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/d561b4afe4e14addb37708472b2ea07e, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/4ff24538ff4e428288740e499ae36eca, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/84f13aafe378489c826b7afabf6b0dd4, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/a2f96124a504430b935eba66e24cca79, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/e52c0814ccf942fab4cf11bd76100872, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/93a983b505074152996323bd7d6c8d00, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/aa60573fcd6343b7825ebc81e381af62, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/7ee5342ff436454b8b89e44cb0c96b65, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/3e90eb72c0fa44d595da746144634104, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/f2f6c41707c74aaaaa98e6f7a67c738e, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/59aa8905b394496cbf17b87fbabb424b, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/72ad68ed977f45a7825c746ac4813094, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/d4720f52842c40109685b8bd4344b28b, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/4f52605b6b144183b8544119e0e14e12, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/0135ce2f1414452eaa260c2232a8d4d7, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/89eb6028a0b5476889dfe3cf71258685, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/f31656a500b14b389dbeea2c912cb547, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/9f750b17c02646248adefcbe4d6414b9, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/69c31daaceea4851bd3a1a7a20e35ab3, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/18757bffe85a4749931bd1bdf4978283, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/74288a0f7b024167a0e4b144ebd5f31b, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/58b13ded44db4ccd8d78d4abdb8eac6b, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/ce13dd6b12f74d20b2daca43df4831a6, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/d74757f1901e4618aaa1cb290c8e2f95, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/22c76c9df31d4d3ba406386ffb7fbcf8, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/eef7d35779bb4034a746303a4d51ff63, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/c63debb455ec4fe9865686c5d57123af] to archive 2024-12-09T10:40:57,520 DEBUG [StoreCloser-TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T10:40:57,526 DEBUG [StoreCloser-TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/0cf007a7125d4686b4d48808b19fa98b to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/0cf007a7125d4686b4d48808b19fa98b 2024-12-09T10:40:57,527 DEBUG [StoreCloser-TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/a9ed01b2d09446af94cac32ee8f8212e to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/a9ed01b2d09446af94cac32ee8f8212e 2024-12-09T10:40:57,528 DEBUG [StoreCloser-TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/4b4ce92d705b4485b69a61cf69095a0f to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/4b4ce92d705b4485b69a61cf69095a0f 2024-12-09T10:40:57,530 DEBUG [StoreCloser-TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/32fe83a127de402ea134cac0a94c1b3e to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/32fe83a127de402ea134cac0a94c1b3e 2024-12-09T10:40:57,531 DEBUG [StoreCloser-TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/996018c21d024fffb1aa2bc5adb98387 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/996018c21d024fffb1aa2bc5adb98387 2024-12-09T10:40:57,532 DEBUG [StoreCloser-TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/fd326f8fa2024083ab10771dc379cacd to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/fd326f8fa2024083ab10771dc379cacd 2024-12-09T10:40:57,534 DEBUG [StoreCloser-TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/519baef860a141f2b3ec677210dad12d to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/519baef860a141f2b3ec677210dad12d 2024-12-09T10:40:57,535 DEBUG [StoreCloser-TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/d561b4afe4e14addb37708472b2ea07e to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/d561b4afe4e14addb37708472b2ea07e 2024-12-09T10:40:57,536 DEBUG [StoreCloser-TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/4ff24538ff4e428288740e499ae36eca to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/4ff24538ff4e428288740e499ae36eca 2024-12-09T10:40:57,537 DEBUG [StoreCloser-TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/84f13aafe378489c826b7afabf6b0dd4 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/84f13aafe378489c826b7afabf6b0dd4 2024-12-09T10:40:57,538 DEBUG [StoreCloser-TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/a2f96124a504430b935eba66e24cca79 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/a2f96124a504430b935eba66e24cca79 2024-12-09T10:40:57,539 DEBUG [StoreCloser-TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/e52c0814ccf942fab4cf11bd76100872 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/e52c0814ccf942fab4cf11bd76100872 2024-12-09T10:40:57,540 DEBUG [StoreCloser-TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/93a983b505074152996323bd7d6c8d00 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/93a983b505074152996323bd7d6c8d00 2024-12-09T10:40:57,541 DEBUG [StoreCloser-TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/aa60573fcd6343b7825ebc81e381af62 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/aa60573fcd6343b7825ebc81e381af62 2024-12-09T10:40:57,543 DEBUG [StoreCloser-TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/7ee5342ff436454b8b89e44cb0c96b65 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/7ee5342ff436454b8b89e44cb0c96b65 2024-12-09T10:40:57,544 DEBUG [StoreCloser-TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/3e90eb72c0fa44d595da746144634104 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/3e90eb72c0fa44d595da746144634104 2024-12-09T10:40:57,545 DEBUG [StoreCloser-TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/f2f6c41707c74aaaaa98e6f7a67c738e to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/f2f6c41707c74aaaaa98e6f7a67c738e 2024-12-09T10:40:57,546 DEBUG [StoreCloser-TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/59aa8905b394496cbf17b87fbabb424b to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/59aa8905b394496cbf17b87fbabb424b 2024-12-09T10:40:57,547 DEBUG [StoreCloser-TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/72ad68ed977f45a7825c746ac4813094 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/72ad68ed977f45a7825c746ac4813094 2024-12-09T10:40:57,548 DEBUG [StoreCloser-TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/d4720f52842c40109685b8bd4344b28b to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/d4720f52842c40109685b8bd4344b28b 2024-12-09T10:40:57,549 DEBUG [StoreCloser-TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/4f52605b6b144183b8544119e0e14e12 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/4f52605b6b144183b8544119e0e14e12 2024-12-09T10:40:57,551 DEBUG [StoreCloser-TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/0135ce2f1414452eaa260c2232a8d4d7 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/0135ce2f1414452eaa260c2232a8d4d7 2024-12-09T10:40:57,552 DEBUG [StoreCloser-TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/89eb6028a0b5476889dfe3cf71258685 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/89eb6028a0b5476889dfe3cf71258685 2024-12-09T10:40:57,553 DEBUG [StoreCloser-TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/f31656a500b14b389dbeea2c912cb547 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/f31656a500b14b389dbeea2c912cb547 2024-12-09T10:40:57,554 DEBUG [StoreCloser-TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/9f750b17c02646248adefcbe4d6414b9 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/9f750b17c02646248adefcbe4d6414b9 2024-12-09T10:40:57,555 DEBUG [StoreCloser-TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/69c31daaceea4851bd3a1a7a20e35ab3 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/69c31daaceea4851bd3a1a7a20e35ab3 2024-12-09T10:40:57,556 DEBUG [StoreCloser-TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/18757bffe85a4749931bd1bdf4978283 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/18757bffe85a4749931bd1bdf4978283 2024-12-09T10:40:57,557 DEBUG [StoreCloser-TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/74288a0f7b024167a0e4b144ebd5f31b to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/74288a0f7b024167a0e4b144ebd5f31b 2024-12-09T10:40:57,558 DEBUG [StoreCloser-TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/58b13ded44db4ccd8d78d4abdb8eac6b to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/58b13ded44db4ccd8d78d4abdb8eac6b 2024-12-09T10:40:57,559 DEBUG [StoreCloser-TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/ce13dd6b12f74d20b2daca43df4831a6 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/ce13dd6b12f74d20b2daca43df4831a6 2024-12-09T10:40:57,560 DEBUG [StoreCloser-TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/d74757f1901e4618aaa1cb290c8e2f95 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/d74757f1901e4618aaa1cb290c8e2f95 2024-12-09T10:40:57,562 DEBUG [StoreCloser-TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/22c76c9df31d4d3ba406386ffb7fbcf8 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/22c76c9df31d4d3ba406386ffb7fbcf8 2024-12-09T10:40:57,563 DEBUG [StoreCloser-TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/eef7d35779bb4034a746303a4d51ff63 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/eef7d35779bb4034a746303a4d51ff63 2024-12-09T10:40:57,565 DEBUG [StoreCloser-TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/c63debb455ec4fe9865686c5d57123af to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/c63debb455ec4fe9865686c5d57123af 2024-12-09T10:40:57,579 DEBUG [StoreCloser-TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/46508c6b9ef0453498d2ce2e61921bf1, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/6f37fb8fc8d04f47ac975c7e6d71de12, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/8540a1ed92f847edb33d38ffa69f8ffe, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/e503f95ccd714dbc8594dccdad97f8a2, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/9a8a4cc395ba48c092c7aa6ad8e215ae, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/a31be429598e44d0a5b6399d21228c95, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/59fa03b684d14f508ad8bd15f9079456, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/68dfe148d84d4c1a9f484a389dd84455, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/dc59d33a81a9401c807c211463f76a9a, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/53cc132cb7e34283ab01482de04ae7da, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/2f73e1188d8b4a598b576b535fa20f34, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/3c6ddfe19ae14e80987a3ff79e738f81, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/ec70f635ad2b4a899042662b2e834420, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/067a265031794c249899bd83c2708289, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/da5ce49da32e4dd99699bba88114ece7, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/a8e0dbea74d64b74aa2ff244839c6072, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/e80496f306ee416cb1605ffa8f3bcc60, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/ecc129019e3e46d49766d4ca5d05c005, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/a800d2a76d15439799e5df23170fe085, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/b3656d6396a94ffdb3e4ce3b0750b9b3, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/64abba655f484942991c9d17ceda5070, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/e30a28c65b274a589bb52c39d979de47, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/a55a0c1d121a4b218bce2d5860448c6e, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/e09aaafaf9e34a5c9a65f13a3945bf0f, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/219d57041a124d99b5b256375c467776, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/d7560fda3d2d4d08b2cef8e478502c82, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/a076f9c734dc4963995bff79b107ce62, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/46a8bb2c8759422e932e70535e94a503, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/5d32ca380ef443d8a047eeb675eab8e3, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/f339c8fba74846d097e6952eaaf3a1b1, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/787e78687c7047eb8fc53206bbb6f359, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/295a6c6a0cba4917b0d3eb84047b6888, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/a234aa97e06b46a3b5c0c52f0af06c4a, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/23545d3781dc4152834aeeb232077612] to archive 2024-12-09T10:40:57,580 DEBUG [StoreCloser-TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T10:40:57,582 DEBUG [StoreCloser-TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/46508c6b9ef0453498d2ce2e61921bf1 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/46508c6b9ef0453498d2ce2e61921bf1 2024-12-09T10:40:57,583 DEBUG [StoreCloser-TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/6f37fb8fc8d04f47ac975c7e6d71de12 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/6f37fb8fc8d04f47ac975c7e6d71de12 2024-12-09T10:40:57,584 DEBUG [StoreCloser-TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/8540a1ed92f847edb33d38ffa69f8ffe to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/8540a1ed92f847edb33d38ffa69f8ffe 2024-12-09T10:40:57,585 DEBUG [StoreCloser-TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/e503f95ccd714dbc8594dccdad97f8a2 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/e503f95ccd714dbc8594dccdad97f8a2 2024-12-09T10:40:57,586 DEBUG [StoreCloser-TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/9a8a4cc395ba48c092c7aa6ad8e215ae to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/9a8a4cc395ba48c092c7aa6ad8e215ae 2024-12-09T10:40:57,587 DEBUG [StoreCloser-TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/a31be429598e44d0a5b6399d21228c95 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/a31be429598e44d0a5b6399d21228c95 2024-12-09T10:40:57,588 DEBUG [StoreCloser-TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/59fa03b684d14f508ad8bd15f9079456 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/59fa03b684d14f508ad8bd15f9079456 2024-12-09T10:40:57,589 DEBUG [StoreCloser-TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/68dfe148d84d4c1a9f484a389dd84455 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/68dfe148d84d4c1a9f484a389dd84455 2024-12-09T10:40:57,590 DEBUG [StoreCloser-TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/dc59d33a81a9401c807c211463f76a9a to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/dc59d33a81a9401c807c211463f76a9a 2024-12-09T10:40:57,591 DEBUG [StoreCloser-TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/53cc132cb7e34283ab01482de04ae7da to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/53cc132cb7e34283ab01482de04ae7da 2024-12-09T10:40:57,592 DEBUG [StoreCloser-TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/2f73e1188d8b4a598b576b535fa20f34 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/2f73e1188d8b4a598b576b535fa20f34 2024-12-09T10:40:57,592 DEBUG [StoreCloser-TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/3c6ddfe19ae14e80987a3ff79e738f81 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/3c6ddfe19ae14e80987a3ff79e738f81 2024-12-09T10:40:57,593 DEBUG [StoreCloser-TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/ec70f635ad2b4a899042662b2e834420 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/ec70f635ad2b4a899042662b2e834420 2024-12-09T10:40:57,594 DEBUG [StoreCloser-TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/067a265031794c249899bd83c2708289 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/067a265031794c249899bd83c2708289 2024-12-09T10:40:57,595 DEBUG [StoreCloser-TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/da5ce49da32e4dd99699bba88114ece7 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/da5ce49da32e4dd99699bba88114ece7 2024-12-09T10:40:57,596 DEBUG [StoreCloser-TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/a8e0dbea74d64b74aa2ff244839c6072 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/a8e0dbea74d64b74aa2ff244839c6072 2024-12-09T10:40:57,597 DEBUG [StoreCloser-TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/e80496f306ee416cb1605ffa8f3bcc60 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/e80496f306ee416cb1605ffa8f3bcc60 2024-12-09T10:40:57,598 DEBUG [StoreCloser-TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/ecc129019e3e46d49766d4ca5d05c005 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/ecc129019e3e46d49766d4ca5d05c005 2024-12-09T10:40:57,599 DEBUG [StoreCloser-TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/a800d2a76d15439799e5df23170fe085 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/a800d2a76d15439799e5df23170fe085 2024-12-09T10:40:57,600 DEBUG [StoreCloser-TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/b3656d6396a94ffdb3e4ce3b0750b9b3 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/b3656d6396a94ffdb3e4ce3b0750b9b3 2024-12-09T10:40:57,601 DEBUG [StoreCloser-TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/64abba655f484942991c9d17ceda5070 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/64abba655f484942991c9d17ceda5070 2024-12-09T10:40:57,602 DEBUG [StoreCloser-TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/e30a28c65b274a589bb52c39d979de47 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/e30a28c65b274a589bb52c39d979de47 2024-12-09T10:40:57,603 DEBUG [StoreCloser-TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/a55a0c1d121a4b218bce2d5860448c6e to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/a55a0c1d121a4b218bce2d5860448c6e 2024-12-09T10:40:57,604 DEBUG [StoreCloser-TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/e09aaafaf9e34a5c9a65f13a3945bf0f to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/e09aaafaf9e34a5c9a65f13a3945bf0f 2024-12-09T10:40:57,604 DEBUG [StoreCloser-TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/219d57041a124d99b5b256375c467776 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/219d57041a124d99b5b256375c467776 2024-12-09T10:40:57,606 DEBUG [StoreCloser-TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/d7560fda3d2d4d08b2cef8e478502c82 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/d7560fda3d2d4d08b2cef8e478502c82 2024-12-09T10:40:57,607 DEBUG [StoreCloser-TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/a076f9c734dc4963995bff79b107ce62 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/a076f9c734dc4963995bff79b107ce62 2024-12-09T10:40:57,608 DEBUG [StoreCloser-TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/46a8bb2c8759422e932e70535e94a503 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/46a8bb2c8759422e932e70535e94a503 2024-12-09T10:40:57,609 DEBUG [StoreCloser-TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/5d32ca380ef443d8a047eeb675eab8e3 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/5d32ca380ef443d8a047eeb675eab8e3 2024-12-09T10:40:57,610 DEBUG [StoreCloser-TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/f339c8fba74846d097e6952eaaf3a1b1 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/f339c8fba74846d097e6952eaaf3a1b1 2024-12-09T10:40:57,611 DEBUG [StoreCloser-TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/787e78687c7047eb8fc53206bbb6f359 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/787e78687c7047eb8fc53206bbb6f359 2024-12-09T10:40:57,612 DEBUG [StoreCloser-TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/295a6c6a0cba4917b0d3eb84047b6888 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/295a6c6a0cba4917b0d3eb84047b6888 2024-12-09T10:40:57,613 DEBUG [StoreCloser-TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/a234aa97e06b46a3b5c0c52f0af06c4a to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/a234aa97e06b46a3b5c0c52f0af06c4a 2024-12-09T10:40:57,614 DEBUG [StoreCloser-TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/23545d3781dc4152834aeeb232077612 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/23545d3781dc4152834aeeb232077612 2024-12-09T10:40:57,615 DEBUG [StoreCloser-TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/b0dc13c0bbab416f838e7e532d23cf8f, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/940aa690831547bb83126b96492c4837, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/d7dea30f31c2466e8eb0c15ec2452db0, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/4d3390a1629343ea906cc28c880e521c, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/a56e848a420b4a5cb42b81474f83fb63, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/6a86fc708dfb4b99926cd2728a25f1d5, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/9549ec226fb14a8c80e95872ee6ff28c, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/ccd5118581644949a62cd9f2b852f010, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/307768823c2544879488b5dd67fe12c6, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/c3b9b04788e64b6e88939f0af5d38970, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/e4b4d95ea682449a9dbeeea3ff98dbf9, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/f033ca332b6e4890b5aae6462dd5ddb5, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/318a8bf8e663499c8bde325f4b071fed, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/00d345660efd4f4f8f3e17405b072809, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/f7ab937338414073bdfd3ec3708b18af, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/753d37e30ca542e28deefc29494bc204, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/b90152d88b684ec9944339d615ad6d39, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/315a02c3b36e435e8c38c8a7ac910e9d, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/614f39bffa314dd9a4d398542db4245c, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/75e17d09da42431cb6836abb3a9a2323, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/2db4c5508f0a4846adcecdef1a641470, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/08070b46c9c94c2e978a3ade9226f7a1, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/29e99b2a489b4f07b3af68651885ba83, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/15d8ad392d81432a8e4928a4741ec520, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/60f6a62fdc5645fe8fb4fd1760787726, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/19422dcb1d1946d0afabfa1e18151058, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/7c2d9230f5d04252b1c75918152a42ea, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/dae218112e1c49df9c43df657b5934f1, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/3f82dbf5e17c42ef85929e8237c0bb7e, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/7f4dcacb7db84579bc1204d1b0db656a, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/9e2e3b4881834b5f96ecac77e3a5b9aa, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/066d197e9720486a95743c60e5f0c633, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/ba211b5061e240d8bac6272d470712db, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/2d477cd8ac9f4d498dae7a8f038d3d07] to archive 2024-12-09T10:40:57,616 DEBUG [StoreCloser-TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T10:40:57,618 DEBUG [StoreCloser-TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/b0dc13c0bbab416f838e7e532d23cf8f to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/b0dc13c0bbab416f838e7e532d23cf8f 2024-12-09T10:40:57,619 DEBUG [StoreCloser-TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/940aa690831547bb83126b96492c4837 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/940aa690831547bb83126b96492c4837 2024-12-09T10:40:57,620 DEBUG [StoreCloser-TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/d7dea30f31c2466e8eb0c15ec2452db0 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/d7dea30f31c2466e8eb0c15ec2452db0 2024-12-09T10:40:57,621 DEBUG [StoreCloser-TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/4d3390a1629343ea906cc28c880e521c to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/4d3390a1629343ea906cc28c880e521c 2024-12-09T10:40:57,622 DEBUG [StoreCloser-TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/a56e848a420b4a5cb42b81474f83fb63 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/a56e848a420b4a5cb42b81474f83fb63 2024-12-09T10:40:57,624 DEBUG [StoreCloser-TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/6a86fc708dfb4b99926cd2728a25f1d5 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/6a86fc708dfb4b99926cd2728a25f1d5 2024-12-09T10:40:57,625 DEBUG [StoreCloser-TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/9549ec226fb14a8c80e95872ee6ff28c to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/9549ec226fb14a8c80e95872ee6ff28c 2024-12-09T10:40:57,626 DEBUG [StoreCloser-TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/ccd5118581644949a62cd9f2b852f010 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/ccd5118581644949a62cd9f2b852f010 2024-12-09T10:40:57,627 DEBUG [StoreCloser-TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/307768823c2544879488b5dd67fe12c6 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/307768823c2544879488b5dd67fe12c6 2024-12-09T10:40:57,628 DEBUG [StoreCloser-TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/c3b9b04788e64b6e88939f0af5d38970 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/c3b9b04788e64b6e88939f0af5d38970 2024-12-09T10:40:57,630 DEBUG [StoreCloser-TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/e4b4d95ea682449a9dbeeea3ff98dbf9 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/e4b4d95ea682449a9dbeeea3ff98dbf9 2024-12-09T10:40:57,631 DEBUG [StoreCloser-TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/f033ca332b6e4890b5aae6462dd5ddb5 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/f033ca332b6e4890b5aae6462dd5ddb5 2024-12-09T10:40:57,632 DEBUG [StoreCloser-TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/318a8bf8e663499c8bde325f4b071fed to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/318a8bf8e663499c8bde325f4b071fed 2024-12-09T10:40:57,633 DEBUG [StoreCloser-TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/00d345660efd4f4f8f3e17405b072809 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/00d345660efd4f4f8f3e17405b072809 2024-12-09T10:40:57,634 DEBUG [StoreCloser-TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/f7ab937338414073bdfd3ec3708b18af to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/f7ab937338414073bdfd3ec3708b18af 2024-12-09T10:40:57,635 DEBUG [StoreCloser-TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/753d37e30ca542e28deefc29494bc204 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/753d37e30ca542e28deefc29494bc204 2024-12-09T10:40:57,636 DEBUG [StoreCloser-TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/b90152d88b684ec9944339d615ad6d39 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/b90152d88b684ec9944339d615ad6d39 2024-12-09T10:40:57,637 DEBUG [StoreCloser-TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/315a02c3b36e435e8c38c8a7ac910e9d to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/315a02c3b36e435e8c38c8a7ac910e9d 2024-12-09T10:40:57,638 DEBUG [StoreCloser-TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/614f39bffa314dd9a4d398542db4245c to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/614f39bffa314dd9a4d398542db4245c 2024-12-09T10:40:57,640 DEBUG [StoreCloser-TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/75e17d09da42431cb6836abb3a9a2323 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/75e17d09da42431cb6836abb3a9a2323 2024-12-09T10:40:57,641 DEBUG [StoreCloser-TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/2db4c5508f0a4846adcecdef1a641470 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/2db4c5508f0a4846adcecdef1a641470 2024-12-09T10:40:57,642 DEBUG [StoreCloser-TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/08070b46c9c94c2e978a3ade9226f7a1 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/08070b46c9c94c2e978a3ade9226f7a1 2024-12-09T10:40:57,643 DEBUG [StoreCloser-TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/29e99b2a489b4f07b3af68651885ba83 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/29e99b2a489b4f07b3af68651885ba83 2024-12-09T10:40:57,644 DEBUG [StoreCloser-TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/15d8ad392d81432a8e4928a4741ec520 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/15d8ad392d81432a8e4928a4741ec520 2024-12-09T10:40:57,645 DEBUG [StoreCloser-TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/60f6a62fdc5645fe8fb4fd1760787726 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/60f6a62fdc5645fe8fb4fd1760787726 2024-12-09T10:40:57,646 DEBUG [StoreCloser-TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/19422dcb1d1946d0afabfa1e18151058 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/19422dcb1d1946d0afabfa1e18151058 2024-12-09T10:40:57,647 DEBUG [StoreCloser-TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/7c2d9230f5d04252b1c75918152a42ea to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/7c2d9230f5d04252b1c75918152a42ea 2024-12-09T10:40:57,648 DEBUG [StoreCloser-TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/dae218112e1c49df9c43df657b5934f1 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/dae218112e1c49df9c43df657b5934f1 2024-12-09T10:40:57,649 DEBUG [StoreCloser-TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/3f82dbf5e17c42ef85929e8237c0bb7e to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/3f82dbf5e17c42ef85929e8237c0bb7e 2024-12-09T10:40:57,651 DEBUG [StoreCloser-TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/7f4dcacb7db84579bc1204d1b0db656a to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/7f4dcacb7db84579bc1204d1b0db656a 2024-12-09T10:40:57,652 DEBUG [StoreCloser-TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/9e2e3b4881834b5f96ecac77e3a5b9aa to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/9e2e3b4881834b5f96ecac77e3a5b9aa 2024-12-09T10:40:57,653 DEBUG [StoreCloser-TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/066d197e9720486a95743c60e5f0c633 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/066d197e9720486a95743c60e5f0c633 2024-12-09T10:40:57,654 DEBUG [StoreCloser-TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/ba211b5061e240d8bac6272d470712db to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/ba211b5061e240d8bac6272d470712db 2024-12-09T10:40:57,655 DEBUG [StoreCloser-TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/2d477cd8ac9f4d498dae7a8f038d3d07 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/2d477cd8ac9f4d498dae7a8f038d3d07 2024-12-09T10:40:57,661 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/recovered.edits/786.seqid, newMaxSeqId=786, maxSeqId=1 2024-12-09T10:40:57,662 INFO [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. 2024-12-09T10:40:57,662 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1676): Region close journal for 1d9d6bba7dcd09663c7b08474fc1c1b5: Waiting for close lock at 1733740856261Running coprocessor pre-close hooks at 1733740856261Disabling compacts and flushes for region at 1733740856261Disabling writes for close at 1733740856261Obtaining lock to block concurrent updates at 1733740856261Preparing flush snapshotting stores in 1d9d6bba7dcd09663c7b08474fc1c1b5 at 1733740856261Finished memstore snapshotting TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1733740856262 (+1 ms)Flushing stores of TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5. at 1733740856263 (+1 ms)Flushing 1d9d6bba7dcd09663c7b08474fc1c1b5/A: creating writer at 1733740856263Flushing 1d9d6bba7dcd09663c7b08474fc1c1b5/A: appending metadata at 1733740856266 (+3 ms)Flushing 1d9d6bba7dcd09663c7b08474fc1c1b5/A: closing flushed file at 1733740856266Flushing 1d9d6bba7dcd09663c7b08474fc1c1b5/B: creating writer at 1733740856676 (+410 ms)Flushing 1d9d6bba7dcd09663c7b08474fc1c1b5/B: appending metadata at 1733740856679 (+3 ms)Flushing 1d9d6bba7dcd09663c7b08474fc1c1b5/B: closing flushed file at 1733740856679Flushing 1d9d6bba7dcd09663c7b08474fc1c1b5/C: creating writer at 1733740857088 (+409 ms)Flushing 1d9d6bba7dcd09663c7b08474fc1c1b5/C: appending metadata at 1733740857091 (+3 ms)Flushing 1d9d6bba7dcd09663c7b08474fc1c1b5/C: closing flushed file at 1733740857091Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@677f90b8: reopening flushed file at 1733740857501 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1b463544: reopening flushed file at 1733740857506 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@633d86dd: reopening flushed file at 1733740857511 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 1d9d6bba7dcd09663c7b08474fc1c1b5 in 1256ms, sequenceid=783, compaction requested=true at 1733740857517 (+6 ms)Writing region close event to WAL at 1733740857656 (+139 ms)Running coprocessor post-close hooks at 1733740857661 (+5 ms)Closed at 1733740857662 (+1 ms) 2024-12-09T10:40:57,664 INFO [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(157): Closed 1d9d6bba7dcd09663c7b08474fc1c1b5 2024-12-09T10:40:57,665 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=1d9d6bba7dcd09663c7b08474fc1c1b5, regionState=CLOSED 2024-12-09T10:40:57,666 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=27, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 because future has completed 2024-12-09T10:40:57,669 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-12-09T10:40:57,669 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; CloseRegionProcedure 1d9d6bba7dcd09663c7b08474fc1c1b5, server=7ea43b39fc24,36101,1733740827678 in 1.5630 sec 2024-12-09T10:40:57,671 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-12-09T10:40:57,671 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1d9d6bba7dcd09663c7b08474fc1c1b5, UNASSIGN in 1.5700 sec 2024-12-09T10:40:57,673 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-12-09T10:40:57,673 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5760 sec 2024-12-09T10:40:57,674 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733740857674"}]},"ts":"1733740857674"} 2024-12-09T10:40:57,676 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-09T10:40:57,676 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-09T10:40:57,677 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5890 sec 2024-12-09T10:40:58,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-09T10:40:58,240 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-09T10:40:58,243 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-12-09T10:40:58,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T10:40:58,249 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=29, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T10:40:58,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-09T10:40:58,250 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=29, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T10:40:58,255 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5 2024-12-09T10:40:58,260 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A, FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B, FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C, FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/recovered.edits] 2024-12-09T10:40:58,263 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/0dd8a387b31f4cc99fc4500ee1427b57 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/0dd8a387b31f4cc99fc4500ee1427b57 2024-12-09T10:40:58,264 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/434156d5247e4194ab7e10504139314a to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/434156d5247e4194ab7e10504139314a 2024-12-09T10:40:58,265 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/ab85214b0d864922a32052afbf486d34 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/ab85214b0d864922a32052afbf486d34 2024-12-09T10:40:58,266 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/fb074d5691f54483b0300bc50f829eb2 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/A/fb074d5691f54483b0300bc50f829eb2 2024-12-09T10:40:58,269 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/0b9d4f28369f45da9a384403dbc5e3f3 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/0b9d4f28369f45da9a384403dbc5e3f3 2024-12-09T10:40:58,271 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/879dfbc72d6343729e1637029b31f4f6 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/879dfbc72d6343729e1637029b31f4f6 2024-12-09T10:40:58,272 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/ab361d23101f4836bfefde171ddbe50e to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/ab361d23101f4836bfefde171ddbe50e 2024-12-09T10:40:58,273 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/d2b6831b24c244ea86451c8b656f9d21 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/B/d2b6831b24c244ea86451c8b656f9d21 2024-12-09T10:40:58,276 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/24051a12bb5b43a58ab13ad2b5c5ff5a to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/24051a12bb5b43a58ab13ad2b5c5ff5a 2024-12-09T10:40:58,277 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/2c34d63b62c44f9e90a0533b12e6cf92 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/2c34d63b62c44f9e90a0533b12e6cf92 2024-12-09T10:40:58,279 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/7206cddfadc448d098ced9b420b7aedb to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/7206cddfadc448d098ced9b420b7aedb 2024-12-09T10:40:58,280 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/810b4b81640648f7879f8a9fa6675128 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/C/810b4b81640648f7879f8a9fa6675128 2024-12-09T10:40:58,283 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/recovered.edits/786.seqid to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5/recovered.edits/786.seqid 2024-12-09T10:40:58,284 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/1d9d6bba7dcd09663c7b08474fc1c1b5 2024-12-09T10:40:58,284 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-09T10:40:58,290 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=29, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T10:40:58,295 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-12-09T10:40:58,303 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-09T10:40:58,340 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-09T10:40:58,342 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=29, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T10:40:58,342 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-09T10:40:58,342 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733740858342"}]},"ts":"9223372036854775807"} 2024-12-09T10:40:58,346 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-09T10:40:58,346 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 1d9d6bba7dcd09663c7b08474fc1c1b5, NAME => 'TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5.', STARTKEY => '', ENDKEY => ''}] 2024-12-09T10:40:58,346 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-09T10:40:58,346 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733740858346"}]},"ts":"9223372036854775807"} 2024-12-09T10:40:58,349 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-09T10:40:58,350 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=29, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T10:40:58,351 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 106 msec 2024-12-09T10:40:58,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-09T10:40:58,361 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-09T10:40:58,361 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-09T10:40:58,374 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=249 (was 211) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_421252992_22 at /127.0.0.1:50650 [Waiting for operation #318] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_421252992_22 at /127.0.0.1:43354 [Waiting for operation #55] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;7ea43b39fc24:36101-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1626372997_22 at /127.0.0.1:55140 [Waiting for operation #334] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=457 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=427 (was 330) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=8571 (was 8650) 2024-12-09T10:40:58,385 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=249, OpenFileDescriptor=457, MaxFileDescriptor=1048576, SystemLoadAverage=427, ProcessCount=11, AvailableMemoryMB=8571 2024-12-09T10:40:58,386 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-09T10:40:58,387 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T10:40:58,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] procedure2.ProcedureExecutor(1139): Stored pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-09T10:40:58,389 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-09T10:40:58,389 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:40:58,389 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 30 2024-12-09T10:40:58,390 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-09T10:40:58,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-12-09T10:40:58,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741951_1127 (size=1061) 2024-12-09T10:40:58,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-12-09T10:40:58,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-12-09T10:40:58,798 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 0fa6aa84c721c187cb4059599cabb328, NAME => 'TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160 2024-12-09T10:40:58,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741952_1128 (size=53) 2024-12-09T10:40:59,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-12-09T10:40:59,205 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T10:40:59,205 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 0fa6aa84c721c187cb4059599cabb328, disabling compactions & flushes 2024-12-09T10:40:59,205 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. 2024-12-09T10:40:59,205 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. 2024-12-09T10:40:59,205 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. after waiting 0 ms 2024-12-09T10:40:59,205 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. 2024-12-09T10:40:59,205 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. 2024-12-09T10:40:59,205 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 0fa6aa84c721c187cb4059599cabb328: Waiting for close lock at 1733740859205Disabling compacts and flushes for region at 1733740859205Disabling writes for close at 1733740859205Writing region close event to WAL at 1733740859205Closed at 1733740859205 2024-12-09T10:40:59,206 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-09T10:40:59,207 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733740859206"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733740859206"}]},"ts":"1733740859206"} 2024-12-09T10:40:59,209 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-09T10:40:59,209 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-09T10:40:59,210 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733740859210"}]},"ts":"1733740859210"} 2024-12-09T10:40:59,211 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-09T10:40:59,212 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0fa6aa84c721c187cb4059599cabb328, ASSIGN}] 2024-12-09T10:40:59,213 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0fa6aa84c721c187cb4059599cabb328, ASSIGN 2024-12-09T10:40:59,213 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0fa6aa84c721c187cb4059599cabb328, ASSIGN; state=OFFLINE, location=7ea43b39fc24,36101,1733740827678; forceNewPlan=false, retain=false 2024-12-09T10:40:59,364 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=0fa6aa84c721c187cb4059599cabb328, regionState=OPENING, regionLocation=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:59,366 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0fa6aa84c721c187cb4059599cabb328, ASSIGN because future has completed 2024-12-09T10:40:59,367 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678}] 2024-12-09T10:40:59,522 INFO [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. 2024-12-09T10:40:59,522 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7752): Opening region: {ENCODED => 0fa6aa84c721c187cb4059599cabb328, NAME => 'TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.', STARTKEY => '', ENDKEY => ''} 2024-12-09T10:40:59,523 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 0fa6aa84c721c187cb4059599cabb328 2024-12-09T10:40:59,523 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T10:40:59,523 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7794): checking encryption for 0fa6aa84c721c187cb4059599cabb328 2024-12-09T10:40:59,523 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7797): checking classloading for 0fa6aa84c721c187cb4059599cabb328 2024-12-09T10:40:59,525 INFO [StoreOpener-0fa6aa84c721c187cb4059599cabb328-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 0fa6aa84c721c187cb4059599cabb328 2024-12-09T10:40:59,526 INFO [StoreOpener-0fa6aa84c721c187cb4059599cabb328-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T10:40:59,527 INFO [StoreOpener-0fa6aa84c721c187cb4059599cabb328-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0fa6aa84c721c187cb4059599cabb328 columnFamilyName A 2024-12-09T10:40:59,527 DEBUG [StoreOpener-0fa6aa84c721c187cb4059599cabb328-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:40:59,527 INFO [StoreOpener-0fa6aa84c721c187cb4059599cabb328-1 {}] regionserver.HStore(327): Store=0fa6aa84c721c187cb4059599cabb328/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T10:40:59,528 INFO [StoreOpener-0fa6aa84c721c187cb4059599cabb328-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 0fa6aa84c721c187cb4059599cabb328 2024-12-09T10:40:59,529 INFO [StoreOpener-0fa6aa84c721c187cb4059599cabb328-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T10:40:59,529 INFO [StoreOpener-0fa6aa84c721c187cb4059599cabb328-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0fa6aa84c721c187cb4059599cabb328 columnFamilyName B 2024-12-09T10:40:59,529 DEBUG [StoreOpener-0fa6aa84c721c187cb4059599cabb328-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:40:59,529 INFO [StoreOpener-0fa6aa84c721c187cb4059599cabb328-1 {}] regionserver.HStore(327): Store=0fa6aa84c721c187cb4059599cabb328/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T10:40:59,529 INFO [StoreOpener-0fa6aa84c721c187cb4059599cabb328-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 0fa6aa84c721c187cb4059599cabb328 2024-12-09T10:40:59,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-12-09T10:40:59,530 INFO [StoreOpener-0fa6aa84c721c187cb4059599cabb328-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T10:40:59,530 INFO [StoreOpener-0fa6aa84c721c187cb4059599cabb328-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0fa6aa84c721c187cb4059599cabb328 columnFamilyName C 2024-12-09T10:40:59,530 DEBUG [StoreOpener-0fa6aa84c721c187cb4059599cabb328-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:40:59,531 INFO [StoreOpener-0fa6aa84c721c187cb4059599cabb328-1 {}] regionserver.HStore(327): Store=0fa6aa84c721c187cb4059599cabb328/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T10:40:59,531 INFO [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. 2024-12-09T10:40:59,531 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1038): replaying wal for 0fa6aa84c721c187cb4059599cabb328 2024-12-09T10:40:59,532 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328 2024-12-09T10:40:59,532 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328 2024-12-09T10:40:59,533 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1048): stopping wal replay for 0fa6aa84c721c187cb4059599cabb328 2024-12-09T10:40:59,533 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1060): Cleaning up temporary data for 0fa6aa84c721c187cb4059599cabb328 2024-12-09T10:40:59,533 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-09T10:40:59,534 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1093): writing seq id for 0fa6aa84c721c187cb4059599cabb328 2024-12-09T10:40:59,536 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T10:40:59,536 INFO [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1114): Opened 0fa6aa84c721c187cb4059599cabb328; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60150702, jitterRate=-0.1036846935749054}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-09T10:40:59,536 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0fa6aa84c721c187cb4059599cabb328 2024-12-09T10:40:59,537 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1006): Region open journal for 0fa6aa84c721c187cb4059599cabb328: Running coprocessor pre-open hook at 1733740859523Writing region info on filesystem at 1733740859523Initializing all the Stores at 1733740859524 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733740859524Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733740859524Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733740859524Cleaning up temporary data from old regions at 1733740859533 (+9 ms)Running coprocessor post-open hooks at 1733740859536 (+3 ms)Region opened successfully at 1733740859537 (+1 ms) 2024-12-09T10:40:59,538 INFO [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., pid=32, masterSystemTime=1733740859519 2024-12-09T10:40:59,540 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. 2024-12-09T10:40:59,540 INFO [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. 2024-12-09T10:40:59,540 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=0fa6aa84c721c187cb4059599cabb328, regionState=OPEN, openSeqNum=2, regionLocation=7ea43b39fc24,36101,1733740827678 2024-12-09T10:40:59,543 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 because future has completed 2024-12-09T10:40:59,545 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-12-09T10:40:59,545 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; OpenRegionProcedure 0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 in 177 msec 2024-12-09T10:40:59,548 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-12-09T10:40:59,548 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0fa6aa84c721c187cb4059599cabb328, ASSIGN in 334 msec 2024-12-09T10:40:59,548 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-09T10:40:59,549 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733740859548"}]},"ts":"1733740859548"} 2024-12-09T10:40:59,551 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-09T10:40:59,552 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-09T10:40:59,553 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1650 sec 2024-12-09T10:41:00,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-12-09T10:41:00,550 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-09T10:41:00,551 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7eea7b7d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:41:00,551 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ea43b39fc24,35095,-1 for getting cluster id 2024-12-09T10:41:00,551 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T10:41:00,553 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0945b5ae-b46b-4b8f-af48-56b0bb92df09' 2024-12-09T10:41:00,553 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T10:41:00,553 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0945b5ae-b46b-4b8f-af48-56b0bb92df09" 2024-12-09T10:41:00,553 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f9e7e14, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:41:00,553 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ea43b39fc24,35095,-1] 2024-12-09T10:41:00,554 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T10:41:00,554 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:41:00,555 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48024, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T10:41:00,556 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16870e56, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:41:00,556 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T10:41:00,557 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ea43b39fc24,36101,1733740827678, seqNum=-1] 2024-12-09T10:41:00,558 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T10:41:00,559 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54182, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T10:41:00,561 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-09T10:41:00,561 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 7ea43b39fc24,35095,1733740826918 2024-12-09T10:41:00,561 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@43c3c82e 2024-12-09T10:41:00,562 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-09T10:41:00,563 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48038, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-09T10:41:00,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-09T10:41:00,569 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T10:41:00,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] procedure2.ProcedureExecutor(1139): Stored pid=33, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-09T10:41:00,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741953_1129 (size=1097) 2024-12-09T10:41:00,995 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-09T10:41:00,995 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-09T10:41:00,999 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-09T10:41:01,006 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0fa6aa84c721c187cb4059599cabb328, REOPEN/MOVE}] 2024-12-09T10:41:01,007 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0fa6aa84c721c187cb4059599cabb328, REOPEN/MOVE 2024-12-09T10:41:01,008 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=0fa6aa84c721c187cb4059599cabb328, regionState=CLOSING, regionLocation=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:01,010 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0fa6aa84c721c187cb4059599cabb328, REOPEN/MOVE because future has completed 2024-12-09T10:41:01,010 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-09T10:41:01,010 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678}] 2024-12-09T10:41:01,164 INFO [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(122): Close 0fa6aa84c721c187cb4059599cabb328 2024-12-09T10:41:01,164 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-09T10:41:01,164 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1722): Closing 0fa6aa84c721c187cb4059599cabb328, disabling compactions & flushes 2024-12-09T10:41:01,164 INFO [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. 2024-12-09T10:41:01,164 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. 2024-12-09T10:41:01,164 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. after waiting 0 ms 2024-12-09T10:41:01,164 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. 2024-12-09T10:41:01,169 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-09T10:41:01,170 INFO [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. 2024-12-09T10:41:01,170 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1676): Region close journal for 0fa6aa84c721c187cb4059599cabb328: Waiting for close lock at 1733740861164Running coprocessor pre-close hooks at 1733740861164Disabling compacts and flushes for region at 1733740861164Disabling writes for close at 1733740861164Writing region close event to WAL at 1733740861165 (+1 ms)Running coprocessor post-close hooks at 1733740861170 (+5 ms)Closed at 1733740861170 2024-12-09T10:41:01,171 WARN [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegionServer(3299): Not adding moved region record: 0fa6aa84c721c187cb4059599cabb328 to self. 2024-12-09T10:41:01,173 INFO [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(157): Closed 0fa6aa84c721c187cb4059599cabb328 2024-12-09T10:41:01,174 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=0fa6aa84c721c187cb4059599cabb328, regionState=CLOSED 2024-12-09T10:41:01,176 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 because future has completed 2024-12-09T10:41:01,180 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-12-09T10:41:01,180 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure 0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 in 167 msec 2024-12-09T10:41:01,181 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0fa6aa84c721c187cb4059599cabb328, REOPEN/MOVE; state=CLOSED, location=7ea43b39fc24,36101,1733740827678; forceNewPlan=false, retain=true 2024-12-09T10:41:01,331 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=0fa6aa84c721c187cb4059599cabb328, regionState=OPENING, regionLocation=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:01,334 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0fa6aa84c721c187cb4059599cabb328, REOPEN/MOVE because future has completed 2024-12-09T10:41:01,334 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678}] 2024-12-09T10:41:01,490 INFO [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. 2024-12-09T10:41:01,490 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7752): Opening region: {ENCODED => 0fa6aa84c721c187cb4059599cabb328, NAME => 'TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.', STARTKEY => '', ENDKEY => ''} 2024-12-09T10:41:01,491 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 0fa6aa84c721c187cb4059599cabb328 2024-12-09T10:41:01,491 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T10:41:01,491 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7794): checking encryption for 0fa6aa84c721c187cb4059599cabb328 2024-12-09T10:41:01,491 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7797): checking classloading for 0fa6aa84c721c187cb4059599cabb328 2024-12-09T10:41:01,497 INFO [StoreOpener-0fa6aa84c721c187cb4059599cabb328-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 0fa6aa84c721c187cb4059599cabb328 2024-12-09T10:41:01,498 INFO [StoreOpener-0fa6aa84c721c187cb4059599cabb328-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T10:41:01,504 INFO [StoreOpener-0fa6aa84c721c187cb4059599cabb328-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0fa6aa84c721c187cb4059599cabb328 columnFamilyName A 2024-12-09T10:41:01,507 DEBUG [StoreOpener-0fa6aa84c721c187cb4059599cabb328-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:01,508 INFO [StoreOpener-0fa6aa84c721c187cb4059599cabb328-1 {}] regionserver.HStore(327): Store=0fa6aa84c721c187cb4059599cabb328/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T10:41:01,508 INFO [StoreOpener-0fa6aa84c721c187cb4059599cabb328-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 0fa6aa84c721c187cb4059599cabb328 2024-12-09T10:41:01,509 INFO [StoreOpener-0fa6aa84c721c187cb4059599cabb328-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T10:41:01,510 INFO [StoreOpener-0fa6aa84c721c187cb4059599cabb328-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0fa6aa84c721c187cb4059599cabb328 columnFamilyName B 2024-12-09T10:41:01,510 DEBUG [StoreOpener-0fa6aa84c721c187cb4059599cabb328-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:01,510 INFO [StoreOpener-0fa6aa84c721c187cb4059599cabb328-1 {}] regionserver.HStore(327): Store=0fa6aa84c721c187cb4059599cabb328/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T10:41:01,510 INFO [StoreOpener-0fa6aa84c721c187cb4059599cabb328-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 0fa6aa84c721c187cb4059599cabb328 2024-12-09T10:41:01,511 INFO [StoreOpener-0fa6aa84c721c187cb4059599cabb328-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T10:41:01,511 INFO [StoreOpener-0fa6aa84c721c187cb4059599cabb328-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0fa6aa84c721c187cb4059599cabb328 columnFamilyName C 2024-12-09T10:41:01,511 DEBUG [StoreOpener-0fa6aa84c721c187cb4059599cabb328-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:01,512 INFO [StoreOpener-0fa6aa84c721c187cb4059599cabb328-1 {}] regionserver.HStore(327): Store=0fa6aa84c721c187cb4059599cabb328/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T10:41:01,512 INFO [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. 2024-12-09T10:41:01,512 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1038): replaying wal for 0fa6aa84c721c187cb4059599cabb328 2024-12-09T10:41:01,513 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328 2024-12-09T10:41:01,515 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328 2024-12-09T10:41:01,516 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1048): stopping wal replay for 0fa6aa84c721c187cb4059599cabb328 2024-12-09T10:41:01,516 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1060): Cleaning up temporary data for 0fa6aa84c721c187cb4059599cabb328 2024-12-09T10:41:01,517 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-09T10:41:01,519 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1093): writing seq id for 0fa6aa84c721c187cb4059599cabb328 2024-12-09T10:41:01,520 INFO [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1114): Opened 0fa6aa84c721c187cb4059599cabb328; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62698684, jitterRate=-0.06571680307388306}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-09T10:41:01,520 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0fa6aa84c721c187cb4059599cabb328 2024-12-09T10:41:01,523 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1006): Region open journal for 0fa6aa84c721c187cb4059599cabb328: Running coprocessor pre-open hook at 1733740861491Writing region info on filesystem at 1733740861491Initializing all the Stores at 1733740861492 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733740861492Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733740861492Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733740861492Cleaning up temporary data from old regions at 1733740861516 (+24 ms)Running coprocessor post-open hooks at 1733740861520 (+4 ms)Region opened successfully at 1733740861523 (+3 ms) 2024-12-09T10:41:01,524 INFO [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., pid=37, masterSystemTime=1733740861486 2024-12-09T10:41:01,527 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. 2024-12-09T10:41:01,527 INFO [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. 2024-12-09T10:41:01,529 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=0fa6aa84c721c187cb4059599cabb328, regionState=OPEN, openSeqNum=5, regionLocation=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:01,531 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 because future has completed 2024-12-09T10:41:01,532 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35095 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=7ea43b39fc24,36101,1733740827678, table=TestAcidGuarantees, region=0fa6aa84c721c187cb4059599cabb328. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-09T10:41:01,535 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=35 2024-12-09T10:41:01,535 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=35, state=SUCCESS, hasLock=false; OpenRegionProcedure 0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 in 198 msec 2024-12-09T10:41:01,537 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-12-09T10:41:01,538 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0fa6aa84c721c187cb4059599cabb328, REOPEN/MOVE in 529 msec 2024-12-09T10:41:01,542 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-12-09T10:41:01,542 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 540 msec 2024-12-09T10:41:01,546 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 971 msec 2024-12-09T10:41:01,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-12-09T10:41:01,548 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-09T10:41:01,549 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47b57f39, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:41:01,549 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ea43b39fc24,35095,-1 for getting cluster id 2024-12-09T10:41:01,550 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T10:41:01,551 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0945b5ae-b46b-4b8f-af48-56b0bb92df09' 2024-12-09T10:41:01,551 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T10:41:01,551 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0945b5ae-b46b-4b8f-af48-56b0bb92df09" 2024-12-09T10:41:01,551 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7dd700f8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:41:01,552 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ea43b39fc24,35095,-1] 2024-12-09T10:41:01,552 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T10:41:01,552 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:41:01,553 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48048, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T10:41:01,554 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78276902, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:41:01,554 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f15a6db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:41:01,555 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ea43b39fc24,35095,-1 for getting cluster id 2024-12-09T10:41:01,555 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T10:41:01,556 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0945b5ae-b46b-4b8f-af48-56b0bb92df09' 2024-12-09T10:41:01,556 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T10:41:01,556 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0945b5ae-b46b-4b8f-af48-56b0bb92df09" 2024-12-09T10:41:01,557 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@579718f6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:41:01,557 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ea43b39fc24,35095,-1] 2024-12-09T10:41:01,557 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T10:41:01,557 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:41:01,558 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48070, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T10:41:01,559 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f289380, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:41:01,560 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@498191e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:41:01,560 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ea43b39fc24,35095,-1 for getting cluster id 2024-12-09T10:41:01,560 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T10:41:01,561 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0945b5ae-b46b-4b8f-af48-56b0bb92df09' 2024-12-09T10:41:01,561 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T10:41:01,561 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0945b5ae-b46b-4b8f-af48-56b0bb92df09" 2024-12-09T10:41:01,561 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@292e9640, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:41:01,562 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ea43b39fc24,35095,-1] 2024-12-09T10:41:01,562 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T10:41:01,562 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:41:01,564 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48090, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T10:41:01,564 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8cfe575, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:41:01,566 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e49d61d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:41:01,566 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ea43b39fc24,35095,-1 for getting cluster id 2024-12-09T10:41:01,566 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T10:41:01,567 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0945b5ae-b46b-4b8f-af48-56b0bb92df09' 2024-12-09T10:41:01,568 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T10:41:01,568 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0945b5ae-b46b-4b8f-af48-56b0bb92df09" 2024-12-09T10:41:01,568 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fbf1cbb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:41:01,568 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ea43b39fc24,35095,-1] 2024-12-09T10:41:01,569 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T10:41:01,569 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:41:01,570 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48104, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T10:41:01,571 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43bd192e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:41:01,572 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@eb41597, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:41:01,572 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ea43b39fc24,35095,-1 for getting cluster id 2024-12-09T10:41:01,572 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T10:41:01,573 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0945b5ae-b46b-4b8f-af48-56b0bb92df09' 2024-12-09T10:41:01,573 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T10:41:01,573 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0945b5ae-b46b-4b8f-af48-56b0bb92df09" 2024-12-09T10:41:01,573 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72b2001e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:41:01,573 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ea43b39fc24,35095,-1] 2024-12-09T10:41:01,574 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T10:41:01,574 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:41:01,575 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48110, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T10:41:01,576 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51b240fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:41:01,578 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c5ef4c2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:41:01,578 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ea43b39fc24,35095,-1 for getting cluster id 2024-12-09T10:41:01,578 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T10:41:01,579 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0945b5ae-b46b-4b8f-af48-56b0bb92df09' 2024-12-09T10:41:01,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T10:41:01,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0945b5ae-b46b-4b8f-af48-56b0bb92df09" 2024-12-09T10:41:01,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bf6ccb2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:41:01,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ea43b39fc24,35095,-1] 2024-12-09T10:41:01,580 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T10:41:01,580 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:41:01,584 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48142, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T10:41:01,585 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20d36ffe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:41:01,589 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@422a8bb7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:41:01,589 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ea43b39fc24,35095,-1 for getting cluster id 2024-12-09T10:41:01,590 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T10:41:01,591 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0945b5ae-b46b-4b8f-af48-56b0bb92df09' 2024-12-09T10:41:01,591 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T10:41:01,591 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0945b5ae-b46b-4b8f-af48-56b0bb92df09" 2024-12-09T10:41:01,591 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2397fb47, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:41:01,591 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ea43b39fc24,35095,-1] 2024-12-09T10:41:01,592 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T10:41:01,592 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:41:01,594 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48158, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T10:41:01,595 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73417116, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:41:01,598 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c2831a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:41:01,598 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ea43b39fc24,35095,-1 for getting cluster id 2024-12-09T10:41:01,598 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T10:41:01,599 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0945b5ae-b46b-4b8f-af48-56b0bb92df09' 2024-12-09T10:41:01,599 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T10:41:01,599 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0945b5ae-b46b-4b8f-af48-56b0bb92df09" 2024-12-09T10:41:01,600 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74023b28, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:41:01,600 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ea43b39fc24,35095,-1] 2024-12-09T10:41:01,600 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T10:41:01,600 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:41:01,602 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48170, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T10:41:01,602 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a3d25d5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:41:01,606 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@428fa770, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:41:01,606 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ea43b39fc24,35095,-1 for getting cluster id 2024-12-09T10:41:01,606 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T10:41:01,607 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0945b5ae-b46b-4b8f-af48-56b0bb92df09' 2024-12-09T10:41:01,607 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T10:41:01,607 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0945b5ae-b46b-4b8f-af48-56b0bb92df09" 2024-12-09T10:41:01,608 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b4ea0a5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:41:01,608 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ea43b39fc24,35095,-1] 2024-12-09T10:41:01,608 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T10:41:01,608 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:41:01,610 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48186, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T10:41:01,611 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7405a174, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:41:01,614 DEBUG [Thread-658 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T10:41:01,616 DEBUG [Thread-662 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T10:41:01,616 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ea43b39fc24,36101,1733740827678, seqNum=-1] 2024-12-09T10:41:01,616 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T10:41:01,618 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54198, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T10:41:01,621 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ea43b39fc24,36101,1733740827678, seqNum=-1] 2024-12-09T10:41:01,621 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T10:41:01,622 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5] 2024-12-09T10:41:01,623 DEBUG [Thread-647 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T10:41:01,624 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ea43b39fc24,36101,1733740827678, seqNum=-1] 2024-12-09T10:41:01,624 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T10:41:01,624 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54206, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T10:41:01,625 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54216, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T10:41:01,627 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5] 2024-12-09T10:41:01,627 DEBUG [Thread-651 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T10:41:01,627 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5] 2024-12-09T10:41:01,628 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ea43b39fc24,36101,1733740827678, seqNum=-1] 2024-12-09T10:41:01,629 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T10:41:01,629 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-09T10:41:01,631 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54232, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T10:41:01,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] procedure2.ProcedureExecutor(1139): Stored pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees 2024-12-09T10:41:01,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-09T10:41:01,633 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T10:41:01,637 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5] 2024-12-09T10:41:01,638 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T10:41:01,638 DEBUG [Thread-656 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T10:41:01,639 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T10:41:01,639 DEBUG [Thread-656 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ea43b39fc24,36101,1733740827678, seqNum=-1] 2024-12-09T10:41:01,639 DEBUG [Thread-656 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T10:41:01,644 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54240, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T10:41:01,646 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5] 2024-12-09T10:41:01,657 DEBUG [Thread-660 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T10:41:01,658 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ea43b39fc24,36101,1733740827678, seqNum=-1] 2024-12-09T10:41:01,659 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T10:41:01,661 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54250, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T10:41:01,661 DEBUG [Thread-649 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T10:41:01,664 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ea43b39fc24,36101,1733740827678, seqNum=-1] 2024-12-09T10:41:01,664 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T10:41:01,670 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5] 2024-12-09T10:41:01,672 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54258, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T10:41:01,678 DEBUG [Thread-653 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T10:41:01,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ea43b39fc24,36101,1733740827678, seqNum=-1] 2024-12-09T10:41:01,680 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5] 2024-12-09T10:41:01,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T10:41:01,681 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54262, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T10:41:01,684 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0fa6aa84c721c187cb4059599cabb328 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-09T10:41:01,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fa6aa84c721c187cb4059599cabb328, store=A 2024-12-09T10:41:01,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:41:01,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fa6aa84c721c187cb4059599cabb328, store=B 2024-12-09T10:41:01,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:41:01,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fa6aa84c721c187cb4059599cabb328, store=C 2024-12-09T10:41:01,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:41:01,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(8855): Flush requested on 0fa6aa84c721c187cb4059599cabb328 2024-12-09T10:41:01,690 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5] 2024-12-09T10:41:01,691 DEBUG [Thread-645 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T10:41:01,692 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ea43b39fc24,36101,1733740827678, seqNum=-1] 2024-12-09T10:41:01,692 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T10:41:01,694 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54272, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T10:41:01,696 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5] 2024-12-09T10:41:01,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-09T10:41:01,771 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120998376236ff40403f94538b4a04975c79_0fa6aa84c721c187cb4059599cabb328 is 50, key is test_row_0/A:col10/1733740861677/Put/seqid=0 2024-12-09T10:41:01,780 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733740861730, max timestamp=1733740861772Num uniques 859; 2024-12-09T10:41:01,780 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T10:41:01,785 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733740861730, max timestamp=1733740861772Num uniques 859; 2024-12-09T10:41:01,785 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T10:41:01,790 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733740861730, max timestamp=1733740861772Num uniques 859; 2024-12-09T10:41:01,790 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T10:41:01,793 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-09T10:41:01,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. 2024-12-09T10:41:01,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. as already flushing 2024-12-09T10:41:01,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. 2024-12-09T10:41:01,793 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 0fa6aa84c721c187cb4059599cabb328, NAME => 'TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:01,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 0fa6aa84c721c187cb4059599cabb328, NAME => 'TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:01,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fa6aa84c721c187cb4059599cabb328, NAME => 'TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fa6aa84c721c187cb4059599cabb328, NAME => 'TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:01,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741954_1130 (size=17034) 2024-12-09T10:41:01,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:01,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54262 deadline: 1733740921837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:01,850 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:01,850 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:01,850 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:01,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54232 deadline: 1733740921837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:01,850 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:01,851 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:01,851 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:01,851 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:01,851 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:01,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54258 deadline: 1733740921837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:01,851 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:01,851 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:01,851 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:01,851 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:01,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54216 deadline: 1733740921837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:01,852 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:01,852 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:01,852 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:01,853 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:01,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54272 deadline: 1733740921838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:01,853 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:01,853 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:01,854 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:01,947 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-09T10:41:01,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. 2024-12-09T10:41:01,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. as already flushing 2024-12-09T10:41:01,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. 2024-12-09T10:41:01,948 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 0fa6aa84c721c187cb4059599cabb328, NAME => 'TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:01,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 0fa6aa84c721c187cb4059599cabb328, NAME => 'TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:01,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fa6aa84c721c187cb4059599cabb328, NAME => 'TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fa6aa84c721c187cb4059599cabb328, NAME => 'TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:01,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-09T10:41:01,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:01,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54262 deadline: 1733740921960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:01,964 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:01,964 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:01,964 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:01,964 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:01,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54216 deadline: 1733740921961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:01,965 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:01,965 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:01,965 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:01,965 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:01,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54232 deadline: 1733740921961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:01,966 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:01,966 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:01,966 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:01,966 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:01,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54272 deadline: 1733740921961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:01,967 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:01,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54258 deadline: 1733740921961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:01,967 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:01,967 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:01,967 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:01,970 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:01,970 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:01,970 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:02,103 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-09T10:41:02,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. 2024-12-09T10:41:02,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. as already flushing 2024-12-09T10:41:02,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. 2024-12-09T10:41:02,103 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 0fa6aa84c721c187cb4059599cabb328, NAME => 'TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:02,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 0fa6aa84c721c187cb4059599cabb328, NAME => 'TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:02,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fa6aa84c721c187cb4059599cabb328, NAME => 'TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fa6aa84c721c187cb4059599cabb328, NAME => 'TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:02,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:02,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54216 deadline: 1733740922170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:02,179 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:02,179 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:02,179 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:02,180 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:02,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54262 deadline: 1733740922170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:02,180 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:02,181 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:02,181 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:02,181 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:02,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54232 deadline: 1733740922170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:02,181 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:02,181 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:02,181 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:02,185 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:02,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54258 deadline: 1733740922180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:02,186 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:02,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54272 deadline: 1733740922180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:02,187 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:02,187 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:02,187 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:02,187 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:02,187 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:02,187 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:02,210 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:02,230 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120998376236ff40403f94538b4a04975c79_0fa6aa84c721c187cb4059599cabb328 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120998376236ff40403f94538b4a04975c79_0fa6aa84c721c187cb4059599cabb328 2024-12-09T10:41:02,231 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/A/bc723ed71c7746e6ba416877d93deb11, store: [table=TestAcidGuarantees family=A region=0fa6aa84c721c187cb4059599cabb328] 2024-12-09T10:41:02,239 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/A/bc723ed71c7746e6ba416877d93deb11 is 175, key is test_row_0/A:col10/1733740861677/Put/seqid=0 2024-12-09T10:41:02,258 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-09T10:41:02,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. 2024-12-09T10:41:02,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. as already flushing 2024-12-09T10:41:02,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. 2024-12-09T10:41:02,258 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 0fa6aa84c721c187cb4059599cabb328, NAME => 'TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:02,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 0fa6aa84c721c187cb4059599cabb328, NAME => 'TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:02,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-09T10:41:02,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fa6aa84c721c187cb4059599cabb328, NAME => 'TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fa6aa84c721c187cb4059599cabb328, NAME => 'TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:02,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741955_1131 (size=48139) 2024-12-09T10:41:02,413 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-09T10:41:02,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. 2024-12-09T10:41:02,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. as already flushing 2024-12-09T10:41:02,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. 2024-12-09T10:41:02,413 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 0fa6aa84c721c187cb4059599cabb328, NAME => 'TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:02,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 0fa6aa84c721c187cb4059599cabb328, NAME => 'TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:02,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fa6aa84c721c187cb4059599cabb328, NAME => 'TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fa6aa84c721c187cb4059599cabb328, NAME => 'TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:02,497 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:02,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54216 deadline: 1733740922494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:02,498 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:02,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54232 deadline: 1733740922494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:02,498 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:02,499 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:02,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54262 deadline: 1733740922494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:02,499 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:02,499 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:02,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:02,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:02,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:02,504 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:02,504 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:02,504 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:02,508 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:02,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54272 deadline: 1733740922500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:02,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:02,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:02,508 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:02,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:02,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54258 deadline: 1733740922500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:02,509 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:02,509 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:02,509 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:02,569 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-09T10:41:02,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. 2024-12-09T10:41:02,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. as already flushing 2024-12-09T10:41:02,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. 2024-12-09T10:41:02,570 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 0fa6aa84c721c187cb4059599cabb328, NAME => 'TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:02,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 0fa6aa84c721c187cb4059599cabb328, NAME => 'TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:02,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fa6aa84c721c187cb4059599cabb328, NAME => 'TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fa6aa84c721c187cb4059599cabb328, NAME => 'TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:02,694 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/A/bc723ed71c7746e6ba416877d93deb11 2024-12-09T10:41:02,723 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-09T10:41:02,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. 2024-12-09T10:41:02,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. as already flushing 2024-12-09T10:41:02,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. 2024-12-09T10:41:02,724 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 0fa6aa84c721c187cb4059599cabb328, NAME => 'TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:02,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 0fa6aa84c721c187cb4059599cabb328, NAME => 'TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:02,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fa6aa84c721c187cb4059599cabb328, NAME => 'TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fa6aa84c721c187cb4059599cabb328, NAME => 'TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:02,731 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/B/87470e4e2fba4d09870884bc4640d990 is 50, key is test_row_0/B:col10/1733740861677/Put/seqid=0 2024-12-09T10:41:02,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-09T10:41:02,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741956_1132 (size=12001) 2024-12-09T10:41:02,878 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-09T10:41:02,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. 2024-12-09T10:41:02,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. as already flushing 2024-12-09T10:41:02,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. 2024-12-09T10:41:02,879 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 0fa6aa84c721c187cb4059599cabb328, NAME => 'TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:02,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 0fa6aa84c721c187cb4059599cabb328, NAME => 'TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:02,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fa6aa84c721c187cb4059599cabb328, NAME => 'TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fa6aa84c721c187cb4059599cabb328, NAME => 'TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:03,015 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:03,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54262 deadline: 1733740923010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:03,016 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:03,016 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:03,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54216 deadline: 1733740923010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:03,016 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:03,016 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:03,016 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:03,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54232 deadline: 1733740923011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:03,018 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:03,018 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:03,018 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:03,020 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:03,020 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:03,020 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:03,029 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:03,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54272 deadline: 1733740923021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:03,029 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:03,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54258 deadline: 1733740923021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:03,030 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:03,030 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:03,030 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:03,031 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:03,031 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:03,031 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:03,032 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-09T10:41:03,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. 2024-12-09T10:41:03,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. as already flushing 2024-12-09T10:41:03,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. 2024-12-09T10:41:03,033 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 0fa6aa84c721c187cb4059599cabb328, NAME => 'TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:03,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 0fa6aa84c721c187cb4059599cabb328, NAME => 'TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:03,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fa6aa84c721c187cb4059599cabb328, NAME => 'TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fa6aa84c721c187cb4059599cabb328, NAME => 'TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:03,184 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/B/87470e4e2fba4d09870884bc4640d990 2024-12-09T10:41:03,188 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-09T10:41:03,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. 2024-12-09T10:41:03,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. as already flushing 2024-12-09T10:41:03,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. 2024-12-09T10:41:03,190 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 0fa6aa84c721c187cb4059599cabb328, NAME => 'TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:03,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 0fa6aa84c721c187cb4059599cabb328, NAME => 'TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:03,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fa6aa84c721c187cb4059599cabb328, NAME => 'TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fa6aa84c721c187cb4059599cabb328, NAME => 'TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:03,218 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-09T10:41:03,225 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/C/b0ecbb1814bf493aaa4d0451511c7a27 is 50, key is test_row_0/C:col10/1733740861677/Put/seqid=0 2024-12-09T10:41:03,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741957_1133 (size=12001) 2024-12-09T10:41:03,264 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/C/b0ecbb1814bf493aaa4d0451511c7a27 2024-12-09T10:41:03,272 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/A/bc723ed71c7746e6ba416877d93deb11 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/bc723ed71c7746e6ba416877d93deb11 2024-12-09T10:41:03,279 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/bc723ed71c7746e6ba416877d93deb11, entries=250, sequenceid=14, filesize=47.0 K 2024-12-09T10:41:03,284 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/B/87470e4e2fba4d09870884bc4640d990 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/87470e4e2fba4d09870884bc4640d990 2024-12-09T10:41:03,301 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/87470e4e2fba4d09870884bc4640d990, entries=150, sequenceid=14, filesize=11.7 K 2024-12-09T10:41:03,302 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/C/b0ecbb1814bf493aaa4d0451511c7a27 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/b0ecbb1814bf493aaa4d0451511c7a27 2024-12-09T10:41:03,329 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/b0ecbb1814bf493aaa4d0451511c7a27, entries=150, sequenceid=14, filesize=11.7 K 2024-12-09T10:41:03,330 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for 0fa6aa84c721c187cb4059599cabb328 in 1646ms, sequenceid=14, compaction requested=false 2024-12-09T10:41:03,331 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0fa6aa84c721c187cb4059599cabb328: 2024-12-09T10:41:03,349 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-09T10:41:03,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. 2024-12-09T10:41:03,350 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2902): Flushing 0fa6aa84c721c187cb4059599cabb328 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-12-09T10:41:03,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fa6aa84c721c187cb4059599cabb328, store=A 2024-12-09T10:41:03,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T10:41:03,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fa6aa84c721c187cb4059599cabb328, store=B 2024-12-09T10:41:03,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T10:41:03,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fa6aa84c721c187cb4059599cabb328, store=C 2024-12-09T10:41:03,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T10:41:03,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412099bd4979dd02b4d269cf58e8a365f2011_0fa6aa84c721c187cb4059599cabb328 is 50, key is test_row_0/A:col10/1733740861809/Put/seqid=0 2024-12-09T10:41:03,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741958_1134 (size=12154) 2024-12-09T10:41:03,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-09T10:41:03,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:03,854 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412099bd4979dd02b4d269cf58e8a365f2011_0fa6aa84c721c187cb4059599cabb328 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412099bd4979dd02b4d269cf58e8a365f2011_0fa6aa84c721c187cb4059599cabb328 2024-12-09T10:41:03,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/A/065f49e028744d5f809432dff47e16b8, store: [table=TestAcidGuarantees family=A region=0fa6aa84c721c187cb4059599cabb328] 2024-12-09T10:41:03,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/A/065f49e028744d5f809432dff47e16b8 is 175, key is test_row_0/A:col10/1733740861809/Put/seqid=0 2024-12-09T10:41:03,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741959_1135 (size=30955) 2024-12-09T10:41:03,884 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=51, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/A/065f49e028744d5f809432dff47e16b8 2024-12-09T10:41:03,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/B/a2496d8710d44be7bce45d45f33129ac is 50, key is test_row_0/B:col10/1733740861809/Put/seqid=0 2024-12-09T10:41:03,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741960_1136 (size=12001) 2024-12-09T10:41:04,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(8855): Flush requested on 0fa6aa84c721c187cb4059599cabb328 2024-12-09T10:41:04,027 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. as already flushing 2024-12-09T10:41:04,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:04,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54232 deadline: 1733740924036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:04,050 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:04,050 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:04,050 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:04,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:04,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54262 deadline: 1733740924038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:04,051 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:04,051 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:04,051 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:04,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:04,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54258 deadline: 1733740924039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:04,052 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:04,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54216 deadline: 1733740924047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:04,052 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 7ea43b39fc24,36101,1733740827678 for 'test_row_1' in 0fa6aa84c721c187cb4059599cabb328 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2216 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T10:41:04,053 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:04,053 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:04,053 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:04,053 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:04,053 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:04,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:04,053 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:04,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54272 deadline: 1733740924050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:04,053 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 7ea43b39fc24,36101,1733740827678 for 'test_row_2' in 0fa6aa84c721c187cb4059599cabb328 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2215 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T10:41:04,054 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:04,054 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:04,054 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:04,163 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:04,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54232 deadline: 1733740924159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:04,164 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:04,164 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:04,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54216 deadline: 1733740924160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:04,164 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:04,164 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:04,164 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:04,164 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:04,164 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:04,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54262 deadline: 1733740924160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:04,164 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:04,165 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:04,165 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:04,165 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:04,313 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/B/a2496d8710d44be7bce45d45f33129ac 2024-12-09T10:41:04,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/C/f6a3d5a1078442b3aaca48d8dae5c589 is 50, key is test_row_0/C:col10/1733740861809/Put/seqid=0 2024-12-09T10:41:04,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741961_1137 (size=12001) 2024-12-09T10:41:04,370 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:04,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54262 deadline: 1733740924370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:04,370 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:04,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54216 deadline: 1733740924370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:04,371 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:04,371 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:04,371 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:04,372 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:04,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:04,372 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:04,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:04,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54232 deadline: 1733740924371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:04,372 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:04,372 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:04,372 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:04,682 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:04,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54216 deadline: 1733740924680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:04,683 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:04,683 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:04,683 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:04,683 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:04,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54262 deadline: 1733740924680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:04,683 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:04,683 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:04,683 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:04,683 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:04,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54232 deadline: 1733740924680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:04,684 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:04,684 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:04,684 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:04,752 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/C/f6a3d5a1078442b3aaca48d8dae5c589 2024-12-09T10:41:04,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/A/065f49e028744d5f809432dff47e16b8 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/065f49e028744d5f809432dff47e16b8 2024-12-09T10:41:04,764 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/065f49e028744d5f809432dff47e16b8, entries=150, sequenceid=51, filesize=30.2 K 2024-12-09T10:41:04,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/B/a2496d8710d44be7bce45d45f33129ac as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/a2496d8710d44be7bce45d45f33129ac 2024-12-09T10:41:04,777 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/a2496d8710d44be7bce45d45f33129ac, entries=150, sequenceid=51, filesize=11.7 K 2024-12-09T10:41:04,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/C/f6a3d5a1078442b3aaca48d8dae5c589 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/f6a3d5a1078442b3aaca48d8dae5c589 2024-12-09T10:41:04,784 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/f6a3d5a1078442b3aaca48d8dae5c589, entries=150, sequenceid=51, filesize=11.7 K 2024-12-09T10:41:04,785 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for 0fa6aa84c721c187cb4059599cabb328 in 1435ms, sequenceid=51, compaction requested=false 2024-12-09T10:41:04,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2603): Flush status journal for 0fa6aa84c721c187cb4059599cabb328: 2024-12-09T10:41:04,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. 2024-12-09T10:41:04,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=39 2024-12-09T10:41:04,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4169): Remote procedure done, pid=39 2024-12-09T10:41:04,789 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-12-09T10:41:04,790 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.1480 sec 2024-12-09T10:41:04,793 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees in 3.1620 sec 2024-12-09T10:41:05,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(8855): Flush requested on 0fa6aa84c721c187cb4059599cabb328 2024-12-09T10:41:05,191 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0fa6aa84c721c187cb4059599cabb328 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-09T10:41:05,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fa6aa84c721c187cb4059599cabb328, store=A 2024-12-09T10:41:05,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:41:05,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fa6aa84c721c187cb4059599cabb328, store=B 2024-12-09T10:41:05,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:41:05,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fa6aa84c721c187cb4059599cabb328, store=C 2024-12-09T10:41:05,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:41:05,206 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209936f247f81ce488ab9b5d80f14186783_0fa6aa84c721c187cb4059599cabb328 is 50, key is test_row_0/A:col10/1733740864036/Put/seqid=0 2024-12-09T10:41:05,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741962_1138 (size=17034) 2024-12-09T10:41:05,241 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:05,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54232 deadline: 1733740925232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:05,242 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:05,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54262 deadline: 1733740925240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:05,242 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:05,242 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:05,242 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:05,243 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733740865223, max timestamp=1733740865240Num uniques 859; 2024-12-09T10:41:05,243 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T10:41:05,243 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733740865223, max timestamp=1733740865240Num uniques 859; 2024-12-09T10:41:05,243 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T10:41:05,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:05,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:05,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:05,246 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733740865223, max timestamp=1733740865240Num uniques 859; 2024-12-09T10:41:05,246 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T10:41:05,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:05,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54216 deadline: 1733740925310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:05,314 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:05,314 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:05,314 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:05,353 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:05,353 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:05,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54232 deadline: 1733740925350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:05,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54262 deadline: 1733740925350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:05,354 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:05,354 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:05,354 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:05,354 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:05,354 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:05,354 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:05,423 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:05,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54216 deadline: 1733740925420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:05,423 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:05,423 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:05,424 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:05,562 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:05,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54232 deadline: 1733740925559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:05,563 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:05,563 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:05,563 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:05,563 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:05,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54262 deadline: 1733740925560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:05,563 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:05,563 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:05,564 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:05,634 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:05,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54216 deadline: 1733740925629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:05,634 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:05,634 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:05,635 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:05,643 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:05,654 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209936f247f81ce488ab9b5d80f14186783_0fa6aa84c721c187cb4059599cabb328 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209936f247f81ce488ab9b5d80f14186783_0fa6aa84c721c187cb4059599cabb328 2024-12-09T10:41:05,655 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/A/83b62a9b2fff451c97a86f0d872e95b8, store: [table=TestAcidGuarantees family=A region=0fa6aa84c721c187cb4059599cabb328] 2024-12-09T10:41:05,656 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/A/83b62a9b2fff451c97a86f0d872e95b8 is 175, key is test_row_0/A:col10/1733740864036/Put/seqid=0 2024-12-09T10:41:05,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741963_1139 (size=48139) 2024-12-09T10:41:05,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-09T10:41:05,800 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T10:41:05,800 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-09T10:41:05,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-12-09T10:41:05,805 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T10:41:05,806 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T10:41:05,806 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T10:41:05,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-09T10:41:05,878 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:05,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54232 deadline: 1733740925870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:05,879 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:05,879 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:05,879 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:05,880 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:05,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54262 deadline: 1733740925870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:05,881 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:05,881 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:05,881 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:05,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-09T10:41:05,946 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:05,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54216 deadline: 1733740925940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:05,947 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:05,947 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:05,947 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:05,961 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-09T10:41:05,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. 2024-12-09T10:41:05,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. as already flushing 2024-12-09T10:41:05,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. 2024-12-09T10:41:05,961 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 0fa6aa84c721c187cb4059599cabb328, NAME => 'TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:05,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 0fa6aa84c721c187cb4059599cabb328, NAME => 'TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:05,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fa6aa84c721c187cb4059599cabb328, NAME => 'TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fa6aa84c721c187cb4059599cabb328, NAME => 'TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:06,070 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:06,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54258 deadline: 1733740926069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:06,070 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:06,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54272 deadline: 1733740926069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:06,070 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 7ea43b39fc24,36101,1733740827678 for 'test_row_1' in 0fa6aa84c721c187cb4059599cabb328 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4235 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T10:41:06,071 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:06,071 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:06,071 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:06,071 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 7ea43b39fc24,36101,1733740827678 for 'test_row_2' in 0fa6aa84c721c187cb4059599cabb328 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4233 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T10:41:06,072 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:06,072 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:06,072 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:06,098 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=62, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/A/83b62a9b2fff451c97a86f0d872e95b8 2024-12-09T10:41:06,115 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-09T10:41:06,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. 2024-12-09T10:41:06,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. as already flushing 2024-12-09T10:41:06,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. 2024-12-09T10:41:06,116 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 0fa6aa84c721c187cb4059599cabb328, NAME => 'TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:06,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 0fa6aa84c721c187cb4059599cabb328, NAME => 'TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:06,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fa6aa84c721c187cb4059599cabb328, NAME => 'TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fa6aa84c721c187cb4059599cabb328, NAME => 'TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:06,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-09T10:41:06,146 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/B/e47aae86e999456c9612f2584ca500e8 is 50, key is test_row_0/B:col10/1733740864036/Put/seqid=0 2024-12-09T10:41:06,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741964_1140 (size=12001) 2024-12-09T10:41:06,170 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/B/e47aae86e999456c9612f2584ca500e8 2024-12-09T10:41:06,193 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/C/8efe3a6694894bb685fc80b4009ce556 is 50, key is test_row_0/C:col10/1733740864036/Put/seqid=0 2024-12-09T10:41:06,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741965_1141 (size=12001) 2024-12-09T10:41:06,231 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/C/8efe3a6694894bb685fc80b4009ce556 2024-12-09T10:41:06,237 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/A/83b62a9b2fff451c97a86f0d872e95b8 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/83b62a9b2fff451c97a86f0d872e95b8 2024-12-09T10:41:06,246 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/83b62a9b2fff451c97a86f0d872e95b8, entries=250, sequenceid=62, filesize=47.0 K 2024-12-09T10:41:06,247 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/B/e47aae86e999456c9612f2584ca500e8 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/e47aae86e999456c9612f2584ca500e8 2024-12-09T10:41:06,255 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/e47aae86e999456c9612f2584ca500e8, entries=150, sequenceid=62, filesize=11.7 K 2024-12-09T10:41:06,259 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/C/8efe3a6694894bb685fc80b4009ce556 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/8efe3a6694894bb685fc80b4009ce556 2024-12-09T10:41:06,269 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-09T10:41:06,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. 2024-12-09T10:41:06,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. as already flushing 2024-12-09T10:41:06,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. 2024-12-09T10:41:06,270 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/8efe3a6694894bb685fc80b4009ce556, entries=150, sequenceid=62, filesize=11.7 K 2024-12-09T10:41:06,270 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 0fa6aa84c721c187cb4059599cabb328, NAME => 'TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:06,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 0fa6aa84c721c187cb4059599cabb328, NAME => 'TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:06,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fa6aa84c721c187cb4059599cabb328, NAME => 'TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fa6aa84c721c187cb4059599cabb328, NAME => 'TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:06,273 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 0fa6aa84c721c187cb4059599cabb328 in 1081ms, sequenceid=62, compaction requested=true 2024-12-09T10:41:06,273 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0fa6aa84c721c187cb4059599cabb328: 2024-12-09T10:41:06,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fa6aa84c721c187cb4059599cabb328:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T10:41:06,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:41:06,274 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T10:41:06,274 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T10:41:06,275 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fa6aa84c721c187cb4059599cabb328:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T10:41:06,275 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:41:06,275 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fa6aa84c721c187cb4059599cabb328:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T10:41:06,275 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T10:41:06,275 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 127233 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T10:41:06,275 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1541): 0fa6aa84c721c187cb4059599cabb328/A is initiating minor compaction (all files) 2024-12-09T10:41:06,275 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fa6aa84c721c187cb4059599cabb328/A in TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. 2024-12-09T10:41:06,276 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/bc723ed71c7746e6ba416877d93deb11, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/065f49e028744d5f809432dff47e16b8, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/83b62a9b2fff451c97a86f0d872e95b8] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp, totalSize=124.3 K 2024-12-09T10:41:06,276 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. 2024-12-09T10:41:06,276 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. files: [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/bc723ed71c7746e6ba416877d93deb11, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/065f49e028744d5f809432dff47e16b8, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/83b62a9b2fff451c97a86f0d872e95b8] 2024-12-09T10:41:06,276 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T10:41:06,276 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1541): 0fa6aa84c721c187cb4059599cabb328/B is initiating minor compaction (all files) 2024-12-09T10:41:06,276 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fa6aa84c721c187cb4059599cabb328/B in TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. 2024-12-09T10:41:06,276 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/87470e4e2fba4d09870884bc4640d990, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/a2496d8710d44be7bce45d45f33129ac, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/e47aae86e999456c9612f2584ca500e8] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp, totalSize=35.2 K 2024-12-09T10:41:06,277 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting bc723ed71c7746e6ba416877d93deb11, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733740861644 2024-12-09T10:41:06,277 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 87470e4e2fba4d09870884bc4640d990, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733740861658 2024-12-09T10:41:06,278 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting a2496d8710d44be7bce45d45f33129ac, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733740861809 2024-12-09T10:41:06,278 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 065f49e028744d5f809432dff47e16b8, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733740861809 2024-12-09T10:41:06,278 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting e47aae86e999456c9612f2584ca500e8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733740864036 2024-12-09T10:41:06,278 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 83b62a9b2fff451c97a86f0d872e95b8, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733740864034 2024-12-09T10:41:06,289 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0fa6aa84c721c187cb4059599cabb328] 2024-12-09T10:41:06,291 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fa6aa84c721c187cb4059599cabb328#B#compaction#124 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:41:06,292 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/B/3f4806145fc54f58b9f0a4583c0cff33 is 50, key is test_row_0/B:col10/1733740864036/Put/seqid=0 2024-12-09T10:41:06,294 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120904c528ecb25e457eb6545f575575e732_0fa6aa84c721c187cb4059599cabb328 store=[table=TestAcidGuarantees family=A region=0fa6aa84c721c187cb4059599cabb328] 2024-12-09T10:41:06,297 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120904c528ecb25e457eb6545f575575e732_0fa6aa84c721c187cb4059599cabb328, store=[table=TestAcidGuarantees family=A region=0fa6aa84c721c187cb4059599cabb328] 2024-12-09T10:41:06,297 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120904c528ecb25e457eb6545f575575e732_0fa6aa84c721c187cb4059599cabb328 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0fa6aa84c721c187cb4059599cabb328] 2024-12-09T10:41:06,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741966_1142 (size=12104) 2024-12-09T10:41:06,331 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/B/3f4806145fc54f58b9f0a4583c0cff33 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/3f4806145fc54f58b9f0a4583c0cff33 2024-12-09T10:41:06,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741967_1143 (size=4469) 2024-12-09T10:41:06,352 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0fa6aa84c721c187cb4059599cabb328/B of 0fa6aa84c721c187cb4059599cabb328 into 3f4806145fc54f58b9f0a4583c0cff33(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:41:06,352 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fa6aa84c721c187cb4059599cabb328: 2024-12-09T10:41:06,352 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., storeName=0fa6aa84c721c187cb4059599cabb328/B, priority=13, startTime=1733740866273; duration=0sec 2024-12-09T10:41:06,352 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T10:41:06,352 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fa6aa84c721c187cb4059599cabb328:B 2024-12-09T10:41:06,352 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T10:41:06,354 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T10:41:06,354 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1541): 0fa6aa84c721c187cb4059599cabb328/C is initiating minor compaction (all files) 2024-12-09T10:41:06,354 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fa6aa84c721c187cb4059599cabb328/C in TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. 2024-12-09T10:41:06,354 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/b0ecbb1814bf493aaa4d0451511c7a27, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/f6a3d5a1078442b3aaca48d8dae5c589, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/8efe3a6694894bb685fc80b4009ce556] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp, totalSize=35.2 K 2024-12-09T10:41:06,355 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting b0ecbb1814bf493aaa4d0451511c7a27, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733740861658 2024-12-09T10:41:06,355 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting f6a3d5a1078442b3aaca48d8dae5c589, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733740861809 2024-12-09T10:41:06,356 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 8efe3a6694894bb685fc80b4009ce556, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733740864036 2024-12-09T10:41:06,379 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fa6aa84c721c187cb4059599cabb328#C#compaction#125 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:41:06,380 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/C/8c584be347c84b2f89f7c5ffe64d8606 is 50, key is test_row_0/C:col10/1733740864036/Put/seqid=0 2024-12-09T10:41:06,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(8855): Flush requested on 0fa6aa84c721c187cb4059599cabb328 2024-12-09T10:41:06,393 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0fa6aa84c721c187cb4059599cabb328 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-12-09T10:41:06,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fa6aa84c721c187cb4059599cabb328, store=A 2024-12-09T10:41:06,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T10:41:06,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fa6aa84c721c187cb4059599cabb328, store=B 2024-12-09T10:41:06,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T10:41:06,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fa6aa84c721c187cb4059599cabb328, store=C 2024-12-09T10:41:06,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T10:41:06,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741968_1144 (size=12104) 2024-12-09T10:41:06,413 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/C/8c584be347c84b2f89f7c5ffe64d8606 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/8c584be347c84b2f89f7c5ffe64d8606 2024-12-09T10:41:06,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-09T10:41:06,430 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-09T10:41:06,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. 2024-12-09T10:41:06,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. as already flushing 2024-12-09T10:41:06,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. 2024-12-09T10:41:06,431 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 0fa6aa84c721c187cb4059599cabb328, NAME => 'TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:06,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 0fa6aa84c721c187cb4059599cabb328, NAME => 'TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:06,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fa6aa84c721c187cb4059599cabb328, NAME => 'TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fa6aa84c721c187cb4059599cabb328, NAME => 'TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:06,441 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0fa6aa84c721c187cb4059599cabb328/C of 0fa6aa84c721c187cb4059599cabb328 into 8c584be347c84b2f89f7c5ffe64d8606(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:41:06,441 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fa6aa84c721c187cb4059599cabb328: 2024-12-09T10:41:06,442 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., storeName=0fa6aa84c721c187cb4059599cabb328/C, priority=13, startTime=1733740866275; duration=0sec 2024-12-09T10:41:06,442 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:41:06,442 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fa6aa84c721c187cb4059599cabb328:C 2024-12-09T10:41:06,462 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412090997665fed9c445b8e1ba18f6b9922d8_0fa6aa84c721c187cb4059599cabb328 is 50, key is test_row_0/A:col10/1733740866391/Put/seqid=0 2024-12-09T10:41:06,468 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:06,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54216 deadline: 1733740926465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:06,469 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:06,469 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:06,469 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:06,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:06,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54262 deadline: 1733740926468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:06,471 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:06,472 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:06,472 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:06,475 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:06,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54232 deadline: 1733740926468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:06,476 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:06,476 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:06,476 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:06,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741969_1145 (size=12154) 2024-12-09T10:41:06,499 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:06,506 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412090997665fed9c445b8e1ba18f6b9922d8_0fa6aa84c721c187cb4059599cabb328 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412090997665fed9c445b8e1ba18f6b9922d8_0fa6aa84c721c187cb4059599cabb328 2024-12-09T10:41:06,507 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/A/42ff5e8e429f425fbc3dbfdb0886d395, store: [table=TestAcidGuarantees family=A region=0fa6aa84c721c187cb4059599cabb328] 2024-12-09T10:41:06,508 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/A/42ff5e8e429f425fbc3dbfdb0886d395 is 175, key is test_row_0/A:col10/1733740866391/Put/seqid=0 2024-12-09T10:41:06,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741970_1146 (size=30955) 2024-12-09T10:41:06,532 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=99, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/A/42ff5e8e429f425fbc3dbfdb0886d395 2024-12-09T10:41:06,548 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/B/1842dc69218641258ea2234ff46b6062 is 50, key is test_row_0/B:col10/1733740866391/Put/seqid=0 2024-12-09T10:41:06,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741971_1147 (size=12001) 2024-12-09T10:41:06,565 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/B/1842dc69218641258ea2234ff46b6062 2024-12-09T10:41:06,580 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/C/3039218d1e594eb0901f4c6de283a328 is 50, key is test_row_0/C:col10/1733740866391/Put/seqid=0 2024-12-09T10:41:06,584 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-09T10:41:06,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. 2024-12-09T10:41:06,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. as already flushing 2024-12-09T10:41:06,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. 2024-12-09T10:41:06,585 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 0fa6aa84c721c187cb4059599cabb328, NAME => 'TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:06,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 0fa6aa84c721c187cb4059599cabb328, NAME => 'TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:06,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fa6aa84c721c187cb4059599cabb328, NAME => 'TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fa6aa84c721c187cb4059599cabb328, NAME => 'TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:06,587 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:06,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54216 deadline: 1733740926581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:06,587 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:06,587 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:06,587 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:06,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54232 deadline: 1733740926582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:06,587 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:06,588 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:06,588 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:06,588 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:06,588 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:06,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54262 deadline: 1733740926582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:06,589 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:06,589 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:06,589 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:06,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741972_1148 (size=12001) 2024-12-09T10:41:06,606 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/C/3039218d1e594eb0901f4c6de283a328 2024-12-09T10:41:06,622 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/A/42ff5e8e429f425fbc3dbfdb0886d395 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/42ff5e8e429f425fbc3dbfdb0886d395 2024-12-09T10:41:06,628 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/42ff5e8e429f425fbc3dbfdb0886d395, entries=150, sequenceid=99, filesize=30.2 K 2024-12-09T10:41:06,630 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/B/1842dc69218641258ea2234ff46b6062 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/1842dc69218641258ea2234ff46b6062 2024-12-09T10:41:06,640 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/1842dc69218641258ea2234ff46b6062, entries=150, sequenceid=99, filesize=11.7 K 2024-12-09T10:41:06,643 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/C/3039218d1e594eb0901f4c6de283a328 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/3039218d1e594eb0901f4c6de283a328 2024-12-09T10:41:06,651 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/3039218d1e594eb0901f4c6de283a328, entries=150, sequenceid=99, filesize=11.7 K 2024-12-09T10:41:06,654 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 0fa6aa84c721c187cb4059599cabb328 in 261ms, sequenceid=99, compaction requested=false 2024-12-09T10:41:06,655 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0fa6aa84c721c187cb4059599cabb328: 2024-12-09T10:41:06,738 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-09T10:41:06,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. 2024-12-09T10:41:06,738 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing 0fa6aa84c721c187cb4059599cabb328 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-09T10:41:06,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fa6aa84c721c187cb4059599cabb328, store=A 2024-12-09T10:41:06,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:41:06,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fa6aa84c721c187cb4059599cabb328, store=B 2024-12-09T10:41:06,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:41:06,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fa6aa84c721c187cb4059599cabb328, store=C 2024-12-09T10:41:06,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:41:06,746 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fa6aa84c721c187cb4059599cabb328#A#compaction#123 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:41:06,747 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/A/020404d7408f4ec8a127f790ef84bce0 is 175, key is test_row_0/A:col10/1733740864036/Put/seqid=0 2024-12-09T10:41:06,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120979be322abbd04fb3b4773e5d7215d0ea_0fa6aa84c721c187cb4059599cabb328 is 50, key is test_row_0/A:col10/1733740866459/Put/seqid=0 2024-12-09T10:41:06,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(8855): Flush requested on 0fa6aa84c721c187cb4059599cabb328 2024-12-09T10:41:06,792 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. as already flushing 2024-12-09T10:41:06,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741973_1149 (size=31058) 2024-12-09T10:41:06,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741974_1150 (size=12154) 2024-12-09T10:41:06,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:06,815 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120979be322abbd04fb3b4773e5d7215d0ea_0fa6aa84c721c187cb4059599cabb328 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120979be322abbd04fb3b4773e5d7215d0ea_0fa6aa84c721c187cb4059599cabb328 2024-12-09T10:41:06,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/A/2b6b3c49586f44ebb4f32a59d05de2a2, store: [table=TestAcidGuarantees family=A region=0fa6aa84c721c187cb4059599cabb328] 2024-12-09T10:41:06,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/A/2b6b3c49586f44ebb4f32a59d05de2a2 is 175, key is test_row_0/A:col10/1733740866459/Put/seqid=0 2024-12-09T10:41:06,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741975_1151 (size=30955) 2024-12-09T10:41:06,870 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733740866848, max timestamp=1733740866866Num uniques 859; 2024-12-09T10:41:06,874 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T10:41:06,876 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733740866856, max timestamp=1733740866866Num uniques 859; 2024-12-09T10:41:06,876 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T10:41:06,879 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733740866856, max timestamp=1733740866866Num uniques 859; 2024-12-09T10:41:06,879 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T10:41:06,931 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:06,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54262 deadline: 1733740926924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:06,932 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:06,932 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:06,932 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:06,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-09T10:41:06,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:06,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54216 deadline: 1733740926930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:06,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:06,942 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:06,942 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:06,942 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:06,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54232 deadline: 1733740926931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:06,942 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:06,942 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:06,942 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:07,044 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:07,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54262 deadline: 1733740927039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:07,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:07,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:07,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:07,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:07,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54216 deadline: 1733740927050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:07,054 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:07,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:07,055 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:07,055 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:07,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54232 deadline: 1733740927050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:07,055 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:07,055 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:07,055 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:07,210 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/A/020404d7408f4ec8a127f790ef84bce0 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/020404d7408f4ec8a127f790ef84bce0 2024-12-09T10:41:07,216 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0fa6aa84c721c187cb4059599cabb328/A of 0fa6aa84c721c187cb4059599cabb328 into 020404d7408f4ec8a127f790ef84bce0(size=30.3 K), total size for store is 60.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:41:07,216 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fa6aa84c721c187cb4059599cabb328: 2024-12-09T10:41:07,216 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., storeName=0fa6aa84c721c187cb4059599cabb328/A, priority=13, startTime=1733740866273; duration=0sec 2024-12-09T10:41:07,216 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:41:07,216 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fa6aa84c721c187cb4059599cabb328:A 2024-12-09T10:41:07,251 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:07,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54262 deadline: 1733740927249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:07,252 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:07,252 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:07,252 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:07,258 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=110, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/A/2b6b3c49586f44ebb4f32a59d05de2a2 2024-12-09T10:41:07,264 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:07,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54216 deadline: 1733740927260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:07,264 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:07,265 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:07,265 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:07,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54232 deadline: 1733740927260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:07,265 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:07,271 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:07,271 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:07,271 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:07,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/B/6069a9d574d84c66a94f2b6f9f3797e5 is 50, key is test_row_0/B:col10/1733740866459/Put/seqid=0 2024-12-09T10:41:07,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741976_1152 (size=12001) 2024-12-09T10:41:07,560 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:07,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54262 deadline: 1733740927559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:07,560 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:07,560 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:07,561 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:07,574 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:07,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54216 deadline: 1733740927570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:07,575 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:07,575 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:07,575 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:07,580 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:07,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54232 deadline: 1733740927579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:07,581 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:07,581 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:07,581 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:07,680 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=110 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/B/6069a9d574d84c66a94f2b6f9f3797e5 2024-12-09T10:41:07,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/C/1440e526f8594bc5b88e2452608ba41a is 50, key is test_row_0/C:col10/1733740866459/Put/seqid=0 2024-12-09T10:41:07,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741977_1153 (size=12001) 2024-12-09T10:41:07,717 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=110 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/C/1440e526f8594bc5b88e2452608ba41a 2024-12-09T10:41:07,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/A/2b6b3c49586f44ebb4f32a59d05de2a2 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/2b6b3c49586f44ebb4f32a59d05de2a2 2024-12-09T10:41:07,731 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/2b6b3c49586f44ebb4f32a59d05de2a2, entries=150, sequenceid=110, filesize=30.2 K 2024-12-09T10:41:07,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/B/6069a9d574d84c66a94f2b6f9f3797e5 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/6069a9d574d84c66a94f2b6f9f3797e5 2024-12-09T10:41:07,741 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/6069a9d574d84c66a94f2b6f9f3797e5, entries=150, sequenceid=110, filesize=11.7 K 2024-12-09T10:41:07,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/C/1440e526f8594bc5b88e2452608ba41a as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/1440e526f8594bc5b88e2452608ba41a 2024-12-09T10:41:07,747 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/1440e526f8594bc5b88e2452608ba41a, entries=150, sequenceid=110, filesize=11.7 K 2024-12-09T10:41:07,748 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=136.42 KB/139690 for 0fa6aa84c721c187cb4059599cabb328 in 1010ms, sequenceid=110, compaction requested=true 2024-12-09T10:41:07,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for 0fa6aa84c721c187cb4059599cabb328: 2024-12-09T10:41:07,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. 2024-12-09T10:41:07,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-12-09T10:41:07,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-12-09T10:41:07,754 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-12-09T10:41:07,754 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9460 sec 2024-12-09T10:41:07,756 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 1.9550 sec 2024-12-09T10:41:07,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-09T10:41:07,949 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T10:41:07,950 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-09T10:41:07,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-12-09T10:41:07,953 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T10:41:07,953 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T10:41:07,954 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T10:41:07,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-09T10:41:08,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-09T10:41:08,080 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733740866919, max timestamp=1733740868075Num uniques 1068; 2024-12-09T10:41:08,080 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-12-09T10:41:08,080 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733740866919, max timestamp=1733740868075Num uniques 1018; 2024-12-09T10:41:08,080 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-09T10:41:08,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(8855): Flush requested on 0fa6aa84c721c187cb4059599cabb328 2024-12-09T10:41:08,081 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0fa6aa84c721c187cb4059599cabb328 3/3 column families, dataSize=64.85 KB heapSize=178.80 KB 2024-12-09T10:41:08,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fa6aa84c721c187cb4059599cabb328, store=A 2024-12-09T10:41:08,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T10:41:08,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fa6aa84c721c187cb4059599cabb328, store=B 2024-12-09T10:41:08,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T10:41:08,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fa6aa84c721c187cb4059599cabb328, store=C 2024-12-09T10:41:08,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-09T10:41:08,107 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-09T10:41:08,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. 2024-12-09T10:41:08,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. as already flushing 2024-12-09T10:41:08,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. 2024-12-09T10:41:08,107 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 0fa6aa84c721c187cb4059599cabb328, NAME => 'TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:08,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 0fa6aa84c721c187cb4059599cabb328, NAME => 'TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:08,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fa6aa84c721c187cb4059599cabb328, NAME => 'TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fa6aa84c721c187cb4059599cabb328, NAME => 'TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:08,119 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209ec7178558de34dcfa9a4a8825cc07e51_0fa6aa84c721c187cb4059599cabb328 is 50, key is test_row_0/A:col10/1733740866919/Put/seqid=0 2024-12-09T10:41:08,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741978_1154 (size=12304) 2024-12-09T10:41:08,156 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:08,163 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209ec7178558de34dcfa9a4a8825cc07e51_0fa6aa84c721c187cb4059599cabb328 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209ec7178558de34dcfa9a4a8825cc07e51_0fa6aa84c721c187cb4059599cabb328 2024-12-09T10:41:08,164 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/A/48da3f0ab8d14e0292ba1d69cc18a75f, store: [table=TestAcidGuarantees family=A region=0fa6aa84c721c187cb4059599cabb328] 2024-12-09T10:41:08,165 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/A/48da3f0ab8d14e0292ba1d69cc18a75f is 175, key is test_row_0/A:col10/1733740866919/Put/seqid=0 2024-12-09T10:41:08,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741979_1155 (size=31105) 2024-12-09T10:41:08,178 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:08,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54232 deadline: 1733740928177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:08,181 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1733740868155, max timestamp=1733740868177Num uniques 859; 2024-12-09T10:41:08,181 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1733740868155, max timestamp=1733740868177Num uniques 859; 2024-12-09T10:41:08,181 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1733740868155, max timestamp=1733740868177Num uniques 859; 2024-12-09T10:41:08,181 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T10:41:08,181 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-09T10:41:08,181 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T10:41:08,181 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:08,181 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:08,181 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:08,250 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:08,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54262 deadline: 1733740928241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:08,250 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:08,250 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:08,250 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:08,255 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:08,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54216 deadline: 1733740928250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:08,255 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:08,256 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:08,256 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:08,262 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-09T10:41:08,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. 2024-12-09T10:41:08,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. as already flushing 2024-12-09T10:41:08,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. 2024-12-09T10:41:08,263 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 0fa6aa84c721c187cb4059599cabb328, NAME => 'TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:08,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 0fa6aa84c721c187cb4059599cabb328, NAME => 'TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:08,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fa6aa84c721c187cb4059599cabb328, NAME => 'TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fa6aa84c721c187cb4059599cabb328, NAME => 'TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:08,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-09T10:41:08,294 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:08,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54232 deadline: 1733740928289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:08,294 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:08,294 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:08,294 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:08,364 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:08,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54262 deadline: 1733740928359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:08,365 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:08,365 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:08,365 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:08,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54216 deadline: 1733740928359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:08,365 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:08,365 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:08,365 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:08,365 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:08,416 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-09T10:41:08,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. 2024-12-09T10:41:08,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. as already flushing 2024-12-09T10:41:08,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. 2024-12-09T10:41:08,417 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 0fa6aa84c721c187cb4059599cabb328, NAME => 'TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:08,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 0fa6aa84c721c187cb4059599cabb328, NAME => 'TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:08,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fa6aa84c721c187cb4059599cabb328, NAME => 'TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fa6aa84c721c187cb4059599cabb328, NAME => 'TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:08,502 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:08,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54232 deadline: 1733740928500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:08,503 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:08,503 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:08,503 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:08,572 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-09T10:41:08,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. 2024-12-09T10:41:08,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. as already flushing 2024-12-09T10:41:08,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. 2024-12-09T10:41:08,572 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 0fa6aa84c721c187cb4059599cabb328, NAME => 'TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:08,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 0fa6aa84c721c187cb4059599cabb328, NAME => 'TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:08,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fa6aa84c721c187cb4059599cabb328, NAME => 'TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fa6aa84c721c187cb4059599cabb328, NAME => 'TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:08,574 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=149, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/A/48da3f0ab8d14e0292ba1d69cc18a75f 2024-12-09T10:41:08,576 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:08,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54216 deadline: 1733740928570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:08,576 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:08,576 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:08,576 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:08,576 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:08,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54262 deadline: 1733740928570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:08,577 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:08,577 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:08,577 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:08,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-09T10:41:08,602 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/B/eff84c229d6549f4b952f5df1d3bbd9b is 50, key is test_row_0/B:col10/1733740866919/Put/seqid=0 2024-12-09T10:41:08,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741980_1156 (size=12151) 2024-12-09T10:41:08,618 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=149 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/B/eff84c229d6549f4b952f5df1d3bbd9b 2024-12-09T10:41:08,639 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/C/a926c57ec3df468b9ff28e3ae92be234 is 50, key is test_row_0/C:col10/1733740866919/Put/seqid=0 2024-12-09T10:41:08,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741981_1157 (size=12151) 2024-12-09T10:41:08,650 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=149 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/C/a926c57ec3df468b9ff28e3ae92be234 2024-12-09T10:41:08,661 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/A/48da3f0ab8d14e0292ba1d69cc18a75f as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/48da3f0ab8d14e0292ba1d69cc18a75f 2024-12-09T10:41:08,669 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/48da3f0ab8d14e0292ba1d69cc18a75f, entries=150, sequenceid=149, filesize=30.4 K 2024-12-09T10:41:08,671 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/B/eff84c229d6549f4b952f5df1d3bbd9b as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/eff84c229d6549f4b952f5df1d3bbd9b 2024-12-09T10:41:08,677 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/eff84c229d6549f4b952f5df1d3bbd9b, entries=150, sequenceid=149, filesize=11.9 K 2024-12-09T10:41:08,678 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/C/a926c57ec3df468b9ff28e3ae92be234 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/a926c57ec3df468b9ff28e3ae92be234 2024-12-09T10:41:08,689 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/a926c57ec3df468b9ff28e3ae92be234, entries=150, sequenceid=149, filesize=11.9 K 2024-12-09T10:41:08,690 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~64.85 KB/66410, heapSize ~178.75 KB/183040, currentSize=123.00 KB/125950 for 0fa6aa84c721c187cb4059599cabb328 in 609ms, sequenceid=149, compaction requested=true 2024-12-09T10:41:08,690 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0fa6aa84c721c187cb4059599cabb328: 2024-12-09T10:41:08,691 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T10:41:08,691 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fa6aa84c721c187cb4059599cabb328:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T10:41:08,691 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:41:08,691 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T10:41:08,691 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fa6aa84c721c187cb4059599cabb328:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T10:41:08,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:41:08,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fa6aa84c721c187cb4059599cabb328:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T10:41:08,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T10:41:08,693 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 124073 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T10:41:08,693 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1541): 0fa6aa84c721c187cb4059599cabb328/A is initiating minor compaction (all files) 2024-12-09T10:41:08,694 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fa6aa84c721c187cb4059599cabb328/A in TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. 2024-12-09T10:41:08,694 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/020404d7408f4ec8a127f790ef84bce0, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/42ff5e8e429f425fbc3dbfdb0886d395, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/2b6b3c49586f44ebb4f32a59d05de2a2, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/48da3f0ab8d14e0292ba1d69cc18a75f] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp, totalSize=121.2 K 2024-12-09T10:41:08,694 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. 2024-12-09T10:41:08,694 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. files: [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/020404d7408f4ec8a127f790ef84bce0, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/42ff5e8e429f425fbc3dbfdb0886d395, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/2b6b3c49586f44ebb4f32a59d05de2a2, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/48da3f0ab8d14e0292ba1d69cc18a75f] 2024-12-09T10:41:08,694 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48257 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T10:41:08,694 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1541): 0fa6aa84c721c187cb4059599cabb328/B is initiating minor compaction (all files) 2024-12-09T10:41:08,694 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fa6aa84c721c187cb4059599cabb328/B in TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. 2024-12-09T10:41:08,694 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/3f4806145fc54f58b9f0a4583c0cff33, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/1842dc69218641258ea2234ff46b6062, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/6069a9d574d84c66a94f2b6f9f3797e5, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/eff84c229d6549f4b952f5df1d3bbd9b] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp, totalSize=47.1 K 2024-12-09T10:41:08,695 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 020404d7408f4ec8a127f790ef84bce0, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733740864036 2024-12-09T10:41:08,695 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f4806145fc54f58b9f0a4583c0cff33, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733740864036 2024-12-09T10:41:08,695 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 42ff5e8e429f425fbc3dbfdb0886d395, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1733740865295 2024-12-09T10:41:08,696 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 1842dc69218641258ea2234ff46b6062, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1733740865295 2024-12-09T10:41:08,696 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b6b3c49586f44ebb4f32a59d05de2a2, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1733740866459 2024-12-09T10:41:08,696 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 6069a9d574d84c66a94f2b6f9f3797e5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1733740866459 2024-12-09T10:41:08,697 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 48da3f0ab8d14e0292ba1d69cc18a75f, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1733740866919 2024-12-09T10:41:08,697 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting eff84c229d6549f4b952f5df1d3bbd9b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1733740866919 2024-12-09T10:41:08,713 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fa6aa84c721c187cb4059599cabb328#B#compaction#135 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:41:08,714 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/B/38f7012ca3c149a7a6beb6a87706e066 is 50, key is test_row_0/B:col10/1733740866919/Put/seqid=0 2024-12-09T10:41:08,726 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-09T10:41:08,726 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0fa6aa84c721c187cb4059599cabb328] 2024-12-09T10:41:08,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. 2024-12-09T10:41:08,727 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing 0fa6aa84c721c187cb4059599cabb328 3/3 column families, dataSize=123.00 KB heapSize=345.53 KB 2024-12-09T10:41:08,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fa6aa84c721c187cb4059599cabb328, store=A 2024-12-09T10:41:08,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T10:41:08,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fa6aa84c721c187cb4059599cabb328, store=B 2024-12-09T10:41:08,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T10:41:08,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fa6aa84c721c187cb4059599cabb328, store=C 2024-12-09T10:41:08,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T10:41:08,745 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120956622712047649da9541bb9028063f86_0fa6aa84c721c187cb4059599cabb328 store=[table=TestAcidGuarantees family=A region=0fa6aa84c721c187cb4059599cabb328] 2024-12-09T10:41:08,749 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120956622712047649da9541bb9028063f86_0fa6aa84c721c187cb4059599cabb328, store=[table=TestAcidGuarantees family=A region=0fa6aa84c721c187cb4059599cabb328] 2024-12-09T10:41:08,749 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120956622712047649da9541bb9028063f86_0fa6aa84c721c187cb4059599cabb328 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0fa6aa84c721c187cb4059599cabb328] 2024-12-09T10:41:08,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741982_1158 (size=12391) 2024-12-09T10:41:08,782 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/B/38f7012ca3c149a7a6beb6a87706e066 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/38f7012ca3c149a7a6beb6a87706e066 2024-12-09T10:41:08,789 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0fa6aa84c721c187cb4059599cabb328/B of 0fa6aa84c721c187cb4059599cabb328 into 38f7012ca3c149a7a6beb6a87706e066(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:41:08,789 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fa6aa84c721c187cb4059599cabb328: 2024-12-09T10:41:08,790 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., storeName=0fa6aa84c721c187cb4059599cabb328/B, priority=12, startTime=1733740868691; duration=0sec 2024-12-09T10:41:08,790 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T10:41:08,790 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fa6aa84c721c187cb4059599cabb328:B 2024-12-09T10:41:08,790 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T10:41:08,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209f236c4c878b84fc3bfb08d2c9e8fa9ba_0fa6aa84c721c187cb4059599cabb328 is 50, key is test_row_0/A:col10/1733740868239/Put/seqid=0 2024-12-09T10:41:08,792 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48257 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T10:41:08,792 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1541): 0fa6aa84c721c187cb4059599cabb328/C is initiating minor compaction (all files) 2024-12-09T10:41:08,792 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fa6aa84c721c187cb4059599cabb328/C in TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. 2024-12-09T10:41:08,792 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/8c584be347c84b2f89f7c5ffe64d8606, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/3039218d1e594eb0901f4c6de283a328, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/1440e526f8594bc5b88e2452608ba41a, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/a926c57ec3df468b9ff28e3ae92be234] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp, totalSize=47.1 K 2024-12-09T10:41:08,792 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c584be347c84b2f89f7c5ffe64d8606, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733740864036 2024-12-09T10:41:08,793 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 3039218d1e594eb0901f4c6de283a328, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1733740865295 2024-12-09T10:41:08,793 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 1440e526f8594bc5b88e2452608ba41a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1733740866459 2024-12-09T10:41:08,794 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting a926c57ec3df468b9ff28e3ae92be234, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1733740866919 2024-12-09T10:41:08,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741983_1159 (size=4469) 2024-12-09T10:41:08,811 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fa6aa84c721c187cb4059599cabb328#A#compaction#136 average throughput is 0.29 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:41:08,812 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/A/3501a6d4b92846adbf898a541db560e1 is 175, key is test_row_0/A:col10/1733740866919/Put/seqid=0 2024-12-09T10:41:08,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(8855): Flush requested on 0fa6aa84c721c187cb4059599cabb328 2024-12-09T10:41:08,814 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. as already flushing 2024-12-09T10:41:08,825 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fa6aa84c721c187cb4059599cabb328#C#compaction#138 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:41:08,826 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/C/6461152a96f94a269f9beff2cd9c976d is 50, key is test_row_0/C:col10/1733740866919/Put/seqid=0 2024-12-09T10:41:08,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741985_1161 (size=31345) 2024-12-09T10:41:08,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741984_1160 (size=12304) 2024-12-09T10:41:08,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:08,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741986_1162 (size=12391) 2024-12-09T10:41:08,857 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209f236c4c878b84fc3bfb08d2c9e8fa9ba_0fa6aa84c721c187cb4059599cabb328 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209f236c4c878b84fc3bfb08d2c9e8fa9ba_0fa6aa84c721c187cb4059599cabb328 2024-12-09T10:41:08,858 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/C/6461152a96f94a269f9beff2cd9c976d as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/6461152a96f94a269f9beff2cd9c976d 2024-12-09T10:41:08,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/A/a1d197bf4e8d42759b05f637fc6637d9, store: [table=TestAcidGuarantees family=A region=0fa6aa84c721c187cb4059599cabb328] 2024-12-09T10:41:08,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/A/a1d197bf4e8d42759b05f637fc6637d9 is 175, key is test_row_0/A:col10/1733740868239/Put/seqid=0 2024-12-09T10:41:08,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741987_1163 (size=31105) 2024-12-09T10:41:08,870 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0fa6aa84c721c187cb4059599cabb328/C of 0fa6aa84c721c187cb4059599cabb328 into 6461152a96f94a269f9beff2cd9c976d(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:41:08,870 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fa6aa84c721c187cb4059599cabb328: 2024-12-09T10:41:08,870 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., storeName=0fa6aa84c721c187cb4059599cabb328/C, priority=12, startTime=1733740868692; duration=0sec 2024-12-09T10:41:08,870 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:41:08,870 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fa6aa84c721c187cb4059599cabb328:C 2024-12-09T10:41:08,872 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:08,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54232 deadline: 1733740928868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:08,873 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:08,873 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:08,873 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:08,885 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:08,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54216 deadline: 1733740928879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:08,886 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:08,886 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:08,886 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:08,891 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:08,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54262 deadline: 1733740928890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:08,893 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:08,893 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:08,893 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:08,980 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:08,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54232 deadline: 1733740928979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:08,980 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:08,981 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:08,981 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:09,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-09T10:41:09,193 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:09,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54232 deadline: 1733740929189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:09,194 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:09,194 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:09,194 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:09,241 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/A/3501a6d4b92846adbf898a541db560e1 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/3501a6d4b92846adbf898a541db560e1 2024-12-09T10:41:09,248 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0fa6aa84c721c187cb4059599cabb328/A of 0fa6aa84c721c187cb4059599cabb328 into 3501a6d4b92846adbf898a541db560e1(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:41:09,248 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fa6aa84c721c187cb4059599cabb328: 2024-12-09T10:41:09,248 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., storeName=0fa6aa84c721c187cb4059599cabb328/A, priority=12, startTime=1733740868690; duration=0sec 2024-12-09T10:41:09,248 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:41:09,248 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fa6aa84c721c187cb4059599cabb328:A 2024-12-09T10:41:09,271 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=183, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/A/a1d197bf4e8d42759b05f637fc6637d9 2024-12-09T10:41:09,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/B/18c3d288358244e1b5726b8bfb2d358a is 50, key is test_row_0/B:col10/1733740868239/Put/seqid=0 2024-12-09T10:41:09,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741988_1164 (size=12151) 2024-12-09T10:41:09,402 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:09,402 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:09,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54216 deadline: 1733740929400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:09,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54262 deadline: 1733740929400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:09,403 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:09,403 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:09,403 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:09,403 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:09,403 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:09,403 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:09,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:09,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54232 deadline: 1733740929500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:09,503 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:09,503 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:09,503 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:09,701 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/B/18c3d288358244e1b5726b8bfb2d358a 2024-12-09T10:41:09,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/C/96e005858a5d45d2a2e13bcf46613134 is 50, key is test_row_0/C:col10/1733740868239/Put/seqid=0 2024-12-09T10:41:09,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741989_1165 (size=12151) 2024-12-09T10:41:10,015 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:10,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54232 deadline: 1733740930010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:10,016 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:10,016 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:10,016 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:10,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-09T10:41:10,102 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:10,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54258 deadline: 1733740930100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:10,103 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:10,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54272 deadline: 1733740930100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:10,103 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 7ea43b39fc24,36101,1733740827678 for 'test_row_1' in 0fa6aa84c721c187cb4059599cabb328 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8267 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T10:41:10,103 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:10,103 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:10,103 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:10,103 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 7ea43b39fc24,36101,1733740827678 for 'test_row_2' in 0fa6aa84c721c187cb4059599cabb328 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8265 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T10:41:10,104 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:10,104 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:10,104 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:10,115 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/C/96e005858a5d45d2a2e13bcf46613134 2024-12-09T10:41:10,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/A/a1d197bf4e8d42759b05f637fc6637d9 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/a1d197bf4e8d42759b05f637fc6637d9 2024-12-09T10:41:10,129 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/a1d197bf4e8d42759b05f637fc6637d9, entries=150, sequenceid=183, filesize=30.4 K 2024-12-09T10:41:10,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/B/18c3d288358244e1b5726b8bfb2d358a as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/18c3d288358244e1b5726b8bfb2d358a 2024-12-09T10:41:10,138 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/18c3d288358244e1b5726b8bfb2d358a, entries=150, sequenceid=183, filesize=11.9 K 2024-12-09T10:41:10,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/C/96e005858a5d45d2a2e13bcf46613134 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/96e005858a5d45d2a2e13bcf46613134 2024-12-09T10:41:10,144 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/96e005858a5d45d2a2e13bcf46613134, entries=150, sequenceid=183, filesize=11.9 K 2024-12-09T10:41:10,146 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~123.00 KB/125950, heapSize ~345.48 KB/353776, currentSize=60.38 KB/61830 for 0fa6aa84c721c187cb4059599cabb328 in 1419ms, sequenceid=183, compaction requested=false 2024-12-09T10:41:10,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for 0fa6aa84c721c187cb4059599cabb328: 2024-12-09T10:41:10,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. 2024-12-09T10:41:10,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-12-09T10:41:10,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-12-09T10:41:10,149 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-12-09T10:41:10,149 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1940 sec 2024-12-09T10:41:10,152 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 2.2000 sec 2024-12-09T10:41:10,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(8855): Flush requested on 0fa6aa84c721c187cb4059599cabb328 2024-12-09T10:41:10,411 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0fa6aa84c721c187cb4059599cabb328 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-09T10:41:10,411 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fa6aa84c721c187cb4059599cabb328, store=A 2024-12-09T10:41:10,412 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:41:10,412 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fa6aa84c721c187cb4059599cabb328, store=B 2024-12-09T10:41:10,412 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:41:10,412 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fa6aa84c721c187cb4059599cabb328, store=C 2024-12-09T10:41:10,412 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:41:10,421 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120925d84e6264844f07a0936306b40f8f91_0fa6aa84c721c187cb4059599cabb328 is 50, key is test_row_0/A:col10/1733740868838/Put/seqid=0 2024-12-09T10:41:10,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741990_1166 (size=14794) 2024-12-09T10:41:10,436 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:10,442 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120925d84e6264844f07a0936306b40f8f91_0fa6aa84c721c187cb4059599cabb328 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120925d84e6264844f07a0936306b40f8f91_0fa6aa84c721c187cb4059599cabb328 2024-12-09T10:41:10,443 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/A/d37dada0d4f5479cb23d328ef5d6a3c1, store: [table=TestAcidGuarantees family=A region=0fa6aa84c721c187cb4059599cabb328] 2024-12-09T10:41:10,444 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/A/d37dada0d4f5479cb23d328ef5d6a3c1 is 175, key is test_row_0/A:col10/1733740868838/Put/seqid=0 2024-12-09T10:41:10,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741991_1167 (size=39749) 2024-12-09T10:41:10,456 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=199, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/A/d37dada0d4f5479cb23d328ef5d6a3c1 2024-12-09T10:41:10,466 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/B/ce268d9901714d71943d9cd673573ce2 is 50, key is test_row_0/B:col10/1733740868838/Put/seqid=0 2024-12-09T10:41:10,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741992_1168 (size=12151) 2024-12-09T10:41:10,475 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:10,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54216 deadline: 1733740930469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:10,475 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:10,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54262 deadline: 1733740930471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:10,476 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:10,476 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:10,476 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:10,476 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:10,476 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:10,476 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:10,479 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/B/ce268d9901714d71943d9cd673573ce2 2024-12-09T10:41:10,490 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/C/6f36b59f111e421087329cd42721f416 is 50, key is test_row_0/C:col10/1733740868838/Put/seqid=0 2024-12-09T10:41:10,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741993_1169 (size=12151) 2024-12-09T10:41:10,519 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/C/6f36b59f111e421087329cd42721f416 2024-12-09T10:41:10,526 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/A/d37dada0d4f5479cb23d328ef5d6a3c1 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/d37dada0d4f5479cb23d328ef5d6a3c1 2024-12-09T10:41:10,532 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/d37dada0d4f5479cb23d328ef5d6a3c1, entries=200, sequenceid=199, filesize=38.8 K 2024-12-09T10:41:10,533 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/B/ce268d9901714d71943d9cd673573ce2 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/ce268d9901714d71943d9cd673573ce2 2024-12-09T10:41:10,537 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/ce268d9901714d71943d9cd673573ce2, entries=150, sequenceid=199, filesize=11.9 K 2024-12-09T10:41:10,538 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/C/6f36b59f111e421087329cd42721f416 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/6f36b59f111e421087329cd42721f416 2024-12-09T10:41:10,545 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/6f36b59f111e421087329cd42721f416, entries=150, sequenceid=199, filesize=11.9 K 2024-12-09T10:41:10,547 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 0fa6aa84c721c187cb4059599cabb328 in 136ms, sequenceid=199, compaction requested=true 2024-12-09T10:41:10,547 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0fa6aa84c721c187cb4059599cabb328: 2024-12-09T10:41:10,547 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T10:41:10,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fa6aa84c721c187cb4059599cabb328:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T10:41:10,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:41:10,547 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T10:41:10,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fa6aa84c721c187cb4059599cabb328:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T10:41:10,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:41:10,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fa6aa84c721c187cb4059599cabb328:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T10:41:10,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T10:41:10,550 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102199 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T10:41:10,550 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1541): 0fa6aa84c721c187cb4059599cabb328/A is initiating minor compaction (all files) 2024-12-09T10:41:10,550 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fa6aa84c721c187cb4059599cabb328/A in TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. 2024-12-09T10:41:10,550 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/3501a6d4b92846adbf898a541db560e1, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/a1d197bf4e8d42759b05f637fc6637d9, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/d37dada0d4f5479cb23d328ef5d6a3c1] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp, totalSize=99.8 K 2024-12-09T10:41:10,550 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. 2024-12-09T10:41:10,550 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. files: [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/3501a6d4b92846adbf898a541db560e1, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/a1d197bf4e8d42759b05f637fc6637d9, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/d37dada0d4f5479cb23d328ef5d6a3c1] 2024-12-09T10:41:10,551 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3501a6d4b92846adbf898a541db560e1, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1733740866919 2024-12-09T10:41:10,551 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T10:41:10,551 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1541): 0fa6aa84c721c187cb4059599cabb328/B is initiating minor compaction (all files) 2024-12-09T10:41:10,551 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fa6aa84c721c187cb4059599cabb328/B in TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. 2024-12-09T10:41:10,552 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/38f7012ca3c149a7a6beb6a87706e066, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/18c3d288358244e1b5726b8bfb2d358a, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/ce268d9901714d71943d9cd673573ce2] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp, totalSize=35.8 K 2024-12-09T10:41:10,552 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting a1d197bf4e8d42759b05f637fc6637d9, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1733740868234 2024-12-09T10:41:10,552 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 38f7012ca3c149a7a6beb6a87706e066, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1733740866919 2024-12-09T10:41:10,553 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting d37dada0d4f5479cb23d328ef5d6a3c1, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1733740868818 2024-12-09T10:41:10,553 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 18c3d288358244e1b5726b8bfb2d358a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1733740868234 2024-12-09T10:41:10,554 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting ce268d9901714d71943d9cd673573ce2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1733740868838 2024-12-09T10:41:10,574 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0fa6aa84c721c187cb4059599cabb328] 2024-12-09T10:41:10,581 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fa6aa84c721c187cb4059599cabb328#B#compaction#145 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:41:10,581 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/B/d78df51593eb49c3944729d10ea66e1b is 50, key is test_row_0/B:col10/1733740868838/Put/seqid=0 2024-12-09T10:41:10,585 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412090853526bc6d044a691ca3bf2e1185547_0fa6aa84c721c187cb4059599cabb328 store=[table=TestAcidGuarantees family=A region=0fa6aa84c721c187cb4059599cabb328] 2024-12-09T10:41:10,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(8855): Flush requested on 0fa6aa84c721c187cb4059599cabb328 2024-12-09T10:41:10,586 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0fa6aa84c721c187cb4059599cabb328 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-09T10:41:10,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fa6aa84c721c187cb4059599cabb328, store=A 2024-12-09T10:41:10,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:41:10,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fa6aa84c721c187cb4059599cabb328, store=B 2024-12-09T10:41:10,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:41:10,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fa6aa84c721c187cb4059599cabb328, store=C 2024-12-09T10:41:10,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:41:10,587 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412090853526bc6d044a691ca3bf2e1185547_0fa6aa84c721c187cb4059599cabb328, store=[table=TestAcidGuarantees family=A region=0fa6aa84c721c187cb4059599cabb328] 2024-12-09T10:41:10,588 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412090853526bc6d044a691ca3bf2e1185547_0fa6aa84c721c187cb4059599cabb328 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0fa6aa84c721c187cb4059599cabb328] 2024-12-09T10:41:10,620 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:10,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54216 deadline: 1733740930612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:10,621 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:10,621 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:10,621 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:10,628 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:10,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54262 deadline: 1733740930621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:10,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:10,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:10,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:10,633 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412093e840cec029f4335a7430a4d057be534_0fa6aa84c721c187cb4059599cabb328 is 50, key is test_row_0/A:col10/1733740870583/Put/seqid=0 2024-12-09T10:41:10,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741995_1171 (size=4469) 2024-12-09T10:41:10,651 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fa6aa84c721c187cb4059599cabb328#A#compaction#144 average throughput is 0.33 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:41:10,651 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/A/82ce835e9a94447ca78a65844390dd50 is 175, key is test_row_0/A:col10/1733740868838/Put/seqid=0 2024-12-09T10:41:10,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741994_1170 (size=12493) 2024-12-09T10:41:10,684 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/B/d78df51593eb49c3944729d10ea66e1b as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/d78df51593eb49c3944729d10ea66e1b 2024-12-09T10:41:10,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741996_1172 (size=14794) 2024-12-09T10:41:10,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741997_1173 (size=31447) 2024-12-09T10:41:10,697 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0fa6aa84c721c187cb4059599cabb328/B of 0fa6aa84c721c187cb4059599cabb328 into d78df51593eb49c3944729d10ea66e1b(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:41:10,697 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fa6aa84c721c187cb4059599cabb328: 2024-12-09T10:41:10,697 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., storeName=0fa6aa84c721c187cb4059599cabb328/B, priority=13, startTime=1733740870547; duration=0sec 2024-12-09T10:41:10,698 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T10:41:10,698 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fa6aa84c721c187cb4059599cabb328:B 2024-12-09T10:41:10,698 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T10:41:10,703 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T10:41:10,703 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1541): 0fa6aa84c721c187cb4059599cabb328/C is initiating minor compaction (all files) 2024-12-09T10:41:10,703 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fa6aa84c721c187cb4059599cabb328/C in TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. 2024-12-09T10:41:10,703 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/6461152a96f94a269f9beff2cd9c976d, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/96e005858a5d45d2a2e13bcf46613134, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/6f36b59f111e421087329cd42721f416] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp, totalSize=35.8 K 2024-12-09T10:41:10,704 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 6461152a96f94a269f9beff2cd9c976d, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1733740866919 2024-12-09T10:41:10,704 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 96e005858a5d45d2a2e13bcf46613134, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1733740868234 2024-12-09T10:41:10,704 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f36b59f111e421087329cd42721f416, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1733740868838 2024-12-09T10:41:10,718 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fa6aa84c721c187cb4059599cabb328#C#compaction#147 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:41:10,719 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/C/9d86a1a56fec43039e9f9eaea02fb98f is 50, key is test_row_0/C:col10/1733740868838/Put/seqid=0 2024-12-09T10:41:10,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741998_1174 (size=12493) 2024-12-09T10:41:10,735 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:10,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54216 deadline: 1733740930729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:10,735 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:10,736 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:10,736 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:10,741 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:10,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54262 deadline: 1733740930740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:10,741 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:10,741 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:10,741 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:10,940 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:10,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54216 deadline: 1733740930939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:10,941 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:10,941 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:10,941 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:10,953 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:10,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54262 deadline: 1733740930950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:10,954 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:10,954 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:10,954 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:11,034 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:11,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54232 deadline: 1733740931029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:11,035 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 7ea43b39fc24,36101,1733740827678 for 'test_row_2' in 0fa6aa84c721c187cb4059599cabb328 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T10:41:11,036 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:11,036 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:11,036 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:11,085 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,089 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412093e840cec029f4335a7430a4d057be534_0fa6aa84c721c187cb4059599cabb328 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412093e840cec029f4335a7430a4d057be534_0fa6aa84c721c187cb4059599cabb328 2024-12-09T10:41:11,091 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/A/6821b37b3d82407b9fdcc9e5aa501492, store: [table=TestAcidGuarantees family=A region=0fa6aa84c721c187cb4059599cabb328] 2024-12-09T10:41:11,091 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/A/6821b37b3d82407b9fdcc9e5aa501492 is 175, key is test_row_0/A:col10/1733740870583/Put/seqid=0 2024-12-09T10:41:11,103 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/A/82ce835e9a94447ca78a65844390dd50 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/82ce835e9a94447ca78a65844390dd50 2024-12-09T10:41:11,111 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0fa6aa84c721c187cb4059599cabb328/A of 0fa6aa84c721c187cb4059599cabb328 into 82ce835e9a94447ca78a65844390dd50(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:41:11,111 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fa6aa84c721c187cb4059599cabb328: 2024-12-09T10:41:11,111 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., storeName=0fa6aa84c721c187cb4059599cabb328/A, priority=13, startTime=1733740870547; duration=0sec 2024-12-09T10:41:11,111 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:41:11,111 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fa6aa84c721c187cb4059599cabb328:A 2024-12-09T10:41:11,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741999_1175 (size=39749) 2024-12-09T10:41:11,113 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=219, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/A/6821b37b3d82407b9fdcc9e5aa501492 2024-12-09T10:41:11,132 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/B/0d9b62035cb04cbfb351669260902a8e is 50, key is test_row_0/B:col10/1733740870583/Put/seqid=0 2024-12-09T10:41:11,141 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/C/9d86a1a56fec43039e9f9eaea02fb98f as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/9d86a1a56fec43039e9f9eaea02fb98f 2024-12-09T10:41:11,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742000_1176 (size=12151) 2024-12-09T10:41:11,147 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=219 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/B/0d9b62035cb04cbfb351669260902a8e 2024-12-09T10:41:11,149 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0fa6aa84c721c187cb4059599cabb328/C of 0fa6aa84c721c187cb4059599cabb328 into 9d86a1a56fec43039e9f9eaea02fb98f(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:41:11,149 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fa6aa84c721c187cb4059599cabb328: 2024-12-09T10:41:11,149 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., storeName=0fa6aa84c721c187cb4059599cabb328/C, priority=13, startTime=1733740870548; duration=0sec 2024-12-09T10:41:11,149 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:41:11,149 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fa6aa84c721c187cb4059599cabb328:C 2024-12-09T10:41:11,158 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/C/0ae236d8169f4ff2ac3b9945a24449e7 is 50, key is test_row_0/C:col10/1733740870583/Put/seqid=0 2024-12-09T10:41:11,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742001_1177 (size=12151) 2024-12-09T10:41:11,172 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=219 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/C/0ae236d8169f4ff2ac3b9945a24449e7 2024-12-09T10:41:11,188 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/A/6821b37b3d82407b9fdcc9e5aa501492 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/6821b37b3d82407b9fdcc9e5aa501492 2024-12-09T10:41:11,194 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/6821b37b3d82407b9fdcc9e5aa501492, entries=200, sequenceid=219, filesize=38.8 K 2024-12-09T10:41:11,197 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/B/0d9b62035cb04cbfb351669260902a8e as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/0d9b62035cb04cbfb351669260902a8e 2024-12-09T10:41:11,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,202 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/0d9b62035cb04cbfb351669260902a8e, entries=150, sequenceid=219, filesize=11.9 K 2024-12-09T10:41:11,203 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/C/0ae236d8169f4ff2ac3b9945a24449e7 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/0ae236d8169f4ff2ac3b9945a24449e7 2024-12-09T10:41:11,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,210 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/0ae236d8169f4ff2ac3b9945a24449e7, entries=150, sequenceid=219, filesize=11.9 K 2024-12-09T10:41:11,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,211 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 0fa6aa84c721c187cb4059599cabb328 in 625ms, sequenceid=219, compaction requested=false 2024-12-09T10:41:11,211 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0fa6aa84c721c187cb4059599cabb328: 2024-12-09T10:41:11,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(8855): Flush requested on 0fa6aa84c721c187cb4059599cabb328 2024-12-09T10:41:11,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,254 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0fa6aa84c721c187cb4059599cabb328 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-09T10:41:11,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fa6aa84c721c187cb4059599cabb328, store=A 2024-12-09T10:41:11,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:41:11,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fa6aa84c721c187cb4059599cabb328, store=B 2024-12-09T10:41:11,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:41:11,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fa6aa84c721c187cb4059599cabb328, store=C 2024-12-09T10:41:11,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:41:11,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,267 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209a17269eb3ba8421ba2e3489a31b97333_0fa6aa84c721c187cb4059599cabb328 is 50, key is test_row_0/A:col10/1733740871253/Put/seqid=0 2024-12-09T10:41:11,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742002_1178 (size=12304) 2024-12-09T10:41:11,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,274 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,280 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209a17269eb3ba8421ba2e3489a31b97333_0fa6aa84c721c187cb4059599cabb328 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209a17269eb3ba8421ba2e3489a31b97333_0fa6aa84c721c187cb4059599cabb328 2024-12-09T10:41:11,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,282 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/A/58c1596b9ee442b99d82568c92184ea2, store: [table=TestAcidGuarantees family=A region=0fa6aa84c721c187cb4059599cabb328] 2024-12-09T10:41:11,282 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/A/58c1596b9ee442b99d82568c92184ea2 is 175, key is test_row_0/A:col10/1733740871253/Put/seqid=0 2024-12-09T10:41:11,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742003_1179 (size=31101) 2024-12-09T10:41:11,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,311 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:11,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54216 deadline: 1733740931310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:11,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,312 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:11,312 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:11,312 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:11,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:11,315 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:11,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54262 deadline: 1733740931311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:11,316 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:11,316 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:11,316 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:11,421 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:11,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54216 deadline: 1733740931419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:11,422 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:11,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54262 deadline: 1733740931419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:11,422 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:11,422 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:11,422 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:11,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:11,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:11,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:11,531 INFO [master/7ea43b39fc24:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-12-09T10:41:11,531 INFO [master/7ea43b39fc24:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-12-09T10:41:11,632 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:11,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54216 deadline: 1733740931630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:11,632 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:11,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54262 deadline: 1733740931630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:11,633 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:11,633 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:11,633 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:11,633 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:11,633 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:11,633 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:11,688 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=235, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/A/58c1596b9ee442b99d82568c92184ea2 2024-12-09T10:41:11,700 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/B/b72f801956bd46fc98b5a8a195194465 is 50, key is test_row_0/B:col10/1733740871253/Put/seqid=0 2024-12-09T10:41:11,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742004_1180 (size=9757) 2024-12-09T10:41:11,943 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:11,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:11,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54262 deadline: 1733740931941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:11,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54216 deadline: 1733740931940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:11,943 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:11,943 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:11,943 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:11,943 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:11,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:11,944 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:12,117 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/B/b72f801956bd46fc98b5a8a195194465 2024-12-09T10:41:12,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-09T10:41:12,120 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T10:41:12,121 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-09T10:41:12,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-12-09T10:41:12,124 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T10:41:12,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-09T10:41:12,124 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T10:41:12,125 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T10:41:12,131 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/C/cfd836c1bff34bfca0311c83b92b5018 is 50, key is test_row_0/C:col10/1733740871253/Put/seqid=0 2024-12-09T10:41:12,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742005_1181 (size=9757) 2024-12-09T10:41:12,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-09T10:41:12,279 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-09T10:41:12,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. 2024-12-09T10:41:12,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. as already flushing 2024-12-09T10:41:12,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. 2024-12-09T10:41:12,280 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 0fa6aa84c721c187cb4059599cabb328, NAME => 'TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:12,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 0fa6aa84c721c187cb4059599cabb328, NAME => 'TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:12,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fa6aa84c721c187cb4059599cabb328, NAME => 'TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fa6aa84c721c187cb4059599cabb328, NAME => 'TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:12,434 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-09T10:41:12,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. 2024-12-09T10:41:12,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. as already flushing 2024-12-09T10:41:12,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. 2024-12-09T10:41:12,435 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 0fa6aa84c721c187cb4059599cabb328, NAME => 'TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:12,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 0fa6aa84c721c187cb4059599cabb328, NAME => 'TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:12,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fa6aa84c721c187cb4059599cabb328, NAME => 'TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fa6aa84c721c187cb4059599cabb328, NAME => 'TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:12,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-09T10:41:12,454 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:12,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54262 deadline: 1733740932450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:12,455 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:12,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:12,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:12,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54216 deadline: 1733740932450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:12,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:12,456 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:12,456 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:12,456 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:12,546 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/C/cfd836c1bff34bfca0311c83b92b5018 2024-12-09T10:41:12,555 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/A/58c1596b9ee442b99d82568c92184ea2 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/58c1596b9ee442b99d82568c92184ea2 2024-12-09T10:41:12,562 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/58c1596b9ee442b99d82568c92184ea2, entries=150, sequenceid=235, filesize=30.4 K 2024-12-09T10:41:12,563 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/B/b72f801956bd46fc98b5a8a195194465 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/b72f801956bd46fc98b5a8a195194465 2024-12-09T10:41:12,574 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/b72f801956bd46fc98b5a8a195194465, entries=100, sequenceid=235, filesize=9.5 K 2024-12-09T10:41:12,577 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/C/cfd836c1bff34bfca0311c83b92b5018 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/cfd836c1bff34bfca0311c83b92b5018 2024-12-09T10:41:12,584 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/cfd836c1bff34bfca0311c83b92b5018, entries=100, sequenceid=235, filesize=9.5 K 2024-12-09T10:41:12,585 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 0fa6aa84c721c187cb4059599cabb328 in 1331ms, sequenceid=235, compaction requested=true 2024-12-09T10:41:12,585 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0fa6aa84c721c187cb4059599cabb328: 2024-12-09T10:41:12,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fa6aa84c721c187cb4059599cabb328:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T10:41:12,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:41:12,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fa6aa84c721c187cb4059599cabb328:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T10:41:12,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T10:41:12,586 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T10:41:12,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fa6aa84c721c187cb4059599cabb328:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T10:41:12,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-09T10:41:12,586 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T10:41:12,588 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-09T10:41:12,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. 2024-12-09T10:41:12,589 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing 0fa6aa84c721c187cb4059599cabb328 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-09T10:41:12,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fa6aa84c721c187cb4059599cabb328, store=A 2024-12-09T10:41:12,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:41:12,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fa6aa84c721c187cb4059599cabb328, store=B 2024-12-09T10:41:12,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:41:12,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fa6aa84c721c187cb4059599cabb328, store=C 2024-12-09T10:41:12,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:41:12,590 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102297 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T10:41:12,590 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34401 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T10:41:12,590 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1541): 0fa6aa84c721c187cb4059599cabb328/A is initiating minor compaction (all files) 2024-12-09T10:41:12,590 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1541): 0fa6aa84c721c187cb4059599cabb328/B is initiating minor compaction (all files) 2024-12-09T10:41:12,590 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fa6aa84c721c187cb4059599cabb328/B in TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. 2024-12-09T10:41:12,590 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/d78df51593eb49c3944729d10ea66e1b, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/0d9b62035cb04cbfb351669260902a8e, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/b72f801956bd46fc98b5a8a195194465] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp, totalSize=33.6 K 2024-12-09T10:41:12,590 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fa6aa84c721c187cb4059599cabb328/A in TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. 2024-12-09T10:41:12,590 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/82ce835e9a94447ca78a65844390dd50, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/6821b37b3d82407b9fdcc9e5aa501492, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/58c1596b9ee442b99d82568c92184ea2] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp, totalSize=99.9 K 2024-12-09T10:41:12,590 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. 2024-12-09T10:41:12,590 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. files: [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/82ce835e9a94447ca78a65844390dd50, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/6821b37b3d82407b9fdcc9e5aa501492, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/58c1596b9ee442b99d82568c92184ea2] 2024-12-09T10:41:12,591 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting d78df51593eb49c3944729d10ea66e1b, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1733740868838 2024-12-09T10:41:12,591 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 82ce835e9a94447ca78a65844390dd50, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1733740868838 2024-12-09T10:41:12,592 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0d9b62035cb04cbfb351669260902a8e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1733740870443 2024-12-09T10:41:12,593 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 6821b37b3d82407b9fdcc9e5aa501492, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1733740870443 2024-12-09T10:41:12,593 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting b72f801956bd46fc98b5a8a195194465, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1733740870610 2024-12-09T10:41:12,593 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 58c1596b9ee442b99d82568c92184ea2, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1733740870610 2024-12-09T10:41:12,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209116a59b43b34490e9af663e8715ba3a3_0fa6aa84c721c187cb4059599cabb328 is 50, key is test_row_0/A:col10/1733740871301/Put/seqid=0 2024-12-09T10:41:12,611 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0fa6aa84c721c187cb4059599cabb328] 2024-12-09T10:41:12,620 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fa6aa84c721c187cb4059599cabb328#B#compaction#155 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:41:12,620 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/B/19da3decfe974d69a605d61478e5dfa2 is 50, key is test_row_0/B:col10/1733740871253/Put/seqid=0 2024-12-09T10:41:12,626 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412099423497be56b480cadd4c2b79a7b48f0_0fa6aa84c721c187cb4059599cabb328 store=[table=TestAcidGuarantees family=A region=0fa6aa84c721c187cb4059599cabb328] 2024-12-09T10:41:12,628 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412099423497be56b480cadd4c2b79a7b48f0_0fa6aa84c721c187cb4059599cabb328, store=[table=TestAcidGuarantees family=A region=0fa6aa84c721c187cb4059599cabb328] 2024-12-09T10:41:12,629 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412099423497be56b480cadd4c2b79a7b48f0_0fa6aa84c721c187cb4059599cabb328 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0fa6aa84c721c187cb4059599cabb328] 2024-12-09T10:41:12,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742006_1182 (size=12304) 2024-12-09T10:41:12,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:12,653 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209116a59b43b34490e9af663e8715ba3a3_0fa6aa84c721c187cb4059599cabb328 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209116a59b43b34490e9af663e8715ba3a3_0fa6aa84c721c187cb4059599cabb328 2024-12-09T10:41:12,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742007_1183 (size=12595) 2024-12-09T10:41:12,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/A/40f11f3552b542039dc14066a37d1a92, store: [table=TestAcidGuarantees family=A region=0fa6aa84c721c187cb4059599cabb328] 2024-12-09T10:41:12,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/A/40f11f3552b542039dc14066a37d1a92 is 175, key is test_row_0/A:col10/1733740871301/Put/seqid=0 2024-12-09T10:41:12,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742008_1184 (size=4469) 2024-12-09T10:41:12,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742009_1185 (size=31105) 2024-12-09T10:41:12,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-09T10:41:13,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(8855): Flush requested on 0fa6aa84c721c187cb4059599cabb328 2024-12-09T10:41:13,053 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. as already flushing 2024-12-09T10:41:13,064 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/B/19da3decfe974d69a605d61478e5dfa2 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/19da3decfe974d69a605d61478e5dfa2 2024-12-09T10:41:13,065 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fa6aa84c721c187cb4059599cabb328#A#compaction#154 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:41:13,065 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/A/3a4dbebfc45f454e9bbcfcd797a67898 is 175, key is test_row_0/A:col10/1733740871253/Put/seqid=0 2024-12-09T10:41:13,068 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=254, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/A/40f11f3552b542039dc14066a37d1a92 2024-12-09T10:41:13,074 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0fa6aa84c721c187cb4059599cabb328/B of 0fa6aa84c721c187cb4059599cabb328 into 19da3decfe974d69a605d61478e5dfa2(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:41:13,074 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fa6aa84c721c187cb4059599cabb328: 2024-12-09T10:41:13,074 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., storeName=0fa6aa84c721c187cb4059599cabb328/B, priority=13, startTime=1733740872585; duration=0sec 2024-12-09T10:41:13,074 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T10:41:13,075 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fa6aa84c721c187cb4059599cabb328:B 2024-12-09T10:41:13,077 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T10:41:13,080 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34401 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T10:41:13,080 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1541): 0fa6aa84c721c187cb4059599cabb328/C is initiating minor compaction (all files) 2024-12-09T10:41:13,080 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fa6aa84c721c187cb4059599cabb328/C in TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. 2024-12-09T10:41:13,080 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/9d86a1a56fec43039e9f9eaea02fb98f, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/0ae236d8169f4ff2ac3b9945a24449e7, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/cfd836c1bff34bfca0311c83b92b5018] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp, totalSize=33.6 K 2024-12-09T10:41:13,080 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9d86a1a56fec43039e9f9eaea02fb98f, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1733740868838 2024-12-09T10:41:13,081 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ae236d8169f4ff2ac3b9945a24449e7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1733740870443 2024-12-09T10:41:13,081 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting cfd836c1bff34bfca0311c83b92b5018, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1733740870610 2024-12-09T10:41:13,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742010_1186 (size=31656) 2024-12-09T10:41:13,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/B/c4ecdc6ece1f4265902e0ff3c4bbbd61 is 50, key is test_row_0/B:col10/1733740871301/Put/seqid=0 2024-12-09T10:41:13,102 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/A/3a4dbebfc45f454e9bbcfcd797a67898 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/3a4dbebfc45f454e9bbcfcd797a67898 2024-12-09T10:41:13,108 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fa6aa84c721c187cb4059599cabb328#C#compaction#157 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:41:13,109 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/C/03eeddf98a654f10b0f01e80caf82eb5 is 50, key is test_row_0/C:col10/1733740871253/Put/seqid=0 2024-12-09T10:41:13,109 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0fa6aa84c721c187cb4059599cabb328/A of 0fa6aa84c721c187cb4059599cabb328 into 3a4dbebfc45f454e9bbcfcd797a67898(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:41:13,109 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fa6aa84c721c187cb4059599cabb328: 2024-12-09T10:41:13,109 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., storeName=0fa6aa84c721c187cb4059599cabb328/A, priority=13, startTime=1733740872585; duration=0sec 2024-12-09T10:41:13,109 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:41:13,109 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fa6aa84c721c187cb4059599cabb328:A 2024-12-09T10:41:13,114 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:13,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54232 deadline: 1733740933109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:13,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742011_1187 (size=12151) 2024-12-09T10:41:13,116 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:13,116 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:13,116 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:13,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742012_1188 (size=12595) 2024-12-09T10:41:13,227 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:13,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54232 deadline: 1733740933220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:13,228 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:13,228 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:13,228 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:13,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-09T10:41:13,446 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:13,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54232 deadline: 1733740933440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:13,446 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:13,446 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:13,446 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:13,464 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:13,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54216 deadline: 1733740933459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:13,464 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:13,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54262 deadline: 1733740933459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:13,465 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 7ea43b39fc24,36101,1733740827678 for 'test_row_1' in 0fa6aa84c721c187cb4059599cabb328 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2154 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T10:41:13,465 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:13,465 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:13,465 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:13,465 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 7ea43b39fc24,36101,1733740827678 for 'test_row_2' in 0fa6aa84c721c187cb4059599cabb328 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2153 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T10:41:13,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:13,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:13,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:13,517 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/B/c4ecdc6ece1f4265902e0ff3c4bbbd61 2024-12-09T10:41:13,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/C/553ba76ed01f4450b7c7bcbfc466dcaf is 50, key is test_row_0/C:col10/1733740871301/Put/seqid=0 2024-12-09T10:41:13,530 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/C/03eeddf98a654f10b0f01e80caf82eb5 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/03eeddf98a654f10b0f01e80caf82eb5 2024-12-09T10:41:13,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742013_1189 (size=12151) 2024-12-09T10:41:13,535 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/C/553ba76ed01f4450b7c7bcbfc466dcaf 2024-12-09T10:41:13,539 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0fa6aa84c721c187cb4059599cabb328/C of 0fa6aa84c721c187cb4059599cabb328 into 03eeddf98a654f10b0f01e80caf82eb5(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:41:13,539 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fa6aa84c721c187cb4059599cabb328: 2024-12-09T10:41:13,539 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., storeName=0fa6aa84c721c187cb4059599cabb328/C, priority=13, startTime=1733740872586; duration=0sec 2024-12-09T10:41:13,539 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:41:13,539 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fa6aa84c721c187cb4059599cabb328:C 2024-12-09T10:41:13,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/A/40f11f3552b542039dc14066a37d1a92 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/40f11f3552b542039dc14066a37d1a92 2024-12-09T10:41:13,555 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/40f11f3552b542039dc14066a37d1a92, entries=150, sequenceid=254, filesize=30.4 K 2024-12-09T10:41:13,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/B/c4ecdc6ece1f4265902e0ff3c4bbbd61 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/c4ecdc6ece1f4265902e0ff3c4bbbd61 2024-12-09T10:41:13,570 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/c4ecdc6ece1f4265902e0ff3c4bbbd61, entries=150, sequenceid=254, filesize=11.9 K 2024-12-09T10:41:13,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/C/553ba76ed01f4450b7c7bcbfc466dcaf as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/553ba76ed01f4450b7c7bcbfc466dcaf 2024-12-09T10:41:13,583 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/553ba76ed01f4450b7c7bcbfc466dcaf, entries=150, sequenceid=254, filesize=11.9 K 2024-12-09T10:41:13,584 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 0fa6aa84c721c187cb4059599cabb328 in 996ms, sequenceid=254, compaction requested=false 2024-12-09T10:41:13,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 0fa6aa84c721c187cb4059599cabb328: 2024-12-09T10:41:13,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. 2024-12-09T10:41:13,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-12-09T10:41:13,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-12-09T10:41:13,588 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-12-09T10:41:13,588 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4620 sec 2024-12-09T10:41:13,590 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 1.4680 sec 2024-12-09T10:41:13,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(8855): Flush requested on 0fa6aa84c721c187cb4059599cabb328 2024-12-09T10:41:13,763 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0fa6aa84c721c187cb4059599cabb328 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-09T10:41:13,763 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fa6aa84c721c187cb4059599cabb328, store=A 2024-12-09T10:41:13,763 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:41:13,763 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fa6aa84c721c187cb4059599cabb328, store=B 2024-12-09T10:41:13,763 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:41:13,763 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fa6aa84c721c187cb4059599cabb328, store=C 2024-12-09T10:41:13,763 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:41:13,772 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209acd8a996e28b4b7f90dd1b45524ded72_0fa6aa84c721c187cb4059599cabb328 is 50, key is test_row_0/A:col10/1733740873761/Put/seqid=0 2024-12-09T10:41:13,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742014_1190 (size=12454) 2024-12-09T10:41:13,833 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:13,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54232 deadline: 1733740933833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:13,834 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:13,834 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:13,834 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:13,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:13,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54232 deadline: 1733740933940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:13,942 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:13,942 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:13,942 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:14,154 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:14,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54232 deadline: 1733740934150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:14,154 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:14,154 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:14,154 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:14,184 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:14,189 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209acd8a996e28b4b7f90dd1b45524ded72_0fa6aa84c721c187cb4059599cabb328 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209acd8a996e28b4b7f90dd1b45524ded72_0fa6aa84c721c187cb4059599cabb328 2024-12-09T10:41:14,190 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/A/49529cf6fe984a33b2f0e31b6b07806f, store: [table=TestAcidGuarantees family=A region=0fa6aa84c721c187cb4059599cabb328] 2024-12-09T10:41:14,191 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/A/49529cf6fe984a33b2f0e31b6b07806f is 175, key is test_row_0/A:col10/1733740873761/Put/seqid=0 2024-12-09T10:41:14,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742015_1191 (size=31255) 2024-12-09T10:41:14,197 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=271, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/A/49529cf6fe984a33b2f0e31b6b07806f 2024-12-09T10:41:14,206 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/B/ef18e1bcd1a5437890c7958856a67cd2 is 50, key is test_row_0/B:col10/1733740873761/Put/seqid=0 2024-12-09T10:41:14,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742016_1192 (size=12301) 2024-12-09T10:41:14,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-09T10:41:14,280 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T10:41:14,280 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-09T10:41:14,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-12-09T10:41:14,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-09T10:41:14,282 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T10:41:14,283 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T10:41:14,283 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T10:41:14,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-09T10:41:14,436 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-09T10:41:14,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. 2024-12-09T10:41:14,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. as already flushing 2024-12-09T10:41:14,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. 2024-12-09T10:41:14,436 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 0fa6aa84c721c187cb4059599cabb328, NAME => 'TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:14,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 0fa6aa84c721c187cb4059599cabb328, NAME => 'TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:14,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fa6aa84c721c187cb4059599cabb328, NAME => 'TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fa6aa84c721c187cb4059599cabb328, NAME => 'TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:14,463 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:14,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54232 deadline: 1733740934459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:14,463 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:14,463 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:14,463 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:14,589 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-09T10:41:14,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. 2024-12-09T10:41:14,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. as already flushing 2024-12-09T10:41:14,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. 2024-12-09T10:41:14,589 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 0fa6aa84c721c187cb4059599cabb328, NAME => 'TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:14,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 0fa6aa84c721c187cb4059599cabb328, NAME => 'TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:14,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fa6aa84c721c187cb4059599cabb328, NAME => 'TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fa6aa84c721c187cb4059599cabb328, NAME => 'TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:14,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-09T10:41:14,614 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/B/ef18e1bcd1a5437890c7958856a67cd2 2024-12-09T10:41:14,622 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/C/c56fd681b92a4616bdd927876145c32a is 50, key is test_row_0/C:col10/1733740873761/Put/seqid=0 2024-12-09T10:41:14,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742017_1193 (size=12301) 2024-12-09T10:41:14,742 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-09T10:41:14,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. 2024-12-09T10:41:14,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. as already flushing 2024-12-09T10:41:14,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. 2024-12-09T10:41:14,743 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 0fa6aa84c721c187cb4059599cabb328, NAME => 'TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:14,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 0fa6aa84c721c187cb4059599cabb328, NAME => 'TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:14,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fa6aa84c721c187cb4059599cabb328, NAME => 'TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fa6aa84c721c187cb4059599cabb328, NAME => 'TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:14,895 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-09T10:41:14,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. 2024-12-09T10:41:14,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. as already flushing 2024-12-09T10:41:14,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. 2024-12-09T10:41:14,896 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 0fa6aa84c721c187cb4059599cabb328, NAME => 'TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:14,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 0fa6aa84c721c187cb4059599cabb328, NAME => 'TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:14,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fa6aa84c721c187cb4059599cabb328, NAME => 'TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fa6aa84c721c187cb4059599cabb328, NAME => 'TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:14,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-09T10:41:14,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:14,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54232 deadline: 1733740934969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:14,972 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:14,972 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:14,972 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:15,028 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/C/c56fd681b92a4616bdd927876145c32a 2024-12-09T10:41:15,034 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/A/49529cf6fe984a33b2f0e31b6b07806f as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/49529cf6fe984a33b2f0e31b6b07806f 2024-12-09T10:41:15,038 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/49529cf6fe984a33b2f0e31b6b07806f, entries=150, sequenceid=271, filesize=30.5 K 2024-12-09T10:41:15,039 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/B/ef18e1bcd1a5437890c7958856a67cd2 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/ef18e1bcd1a5437890c7958856a67cd2 2024-12-09T10:41:15,044 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/ef18e1bcd1a5437890c7958856a67cd2, entries=150, sequenceid=271, filesize=12.0 K 2024-12-09T10:41:15,044 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/C/c56fd681b92a4616bdd927876145c32a as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/c56fd681b92a4616bdd927876145c32a 2024-12-09T10:41:15,048 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/c56fd681b92a4616bdd927876145c32a, entries=150, sequenceid=271, filesize=12.0 K 2024-12-09T10:41:15,049 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-09T10:41:15,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. 2024-12-09T10:41:15,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. as already flushing 2024-12-09T10:41:15,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. 2024-12-09T10:41:15,049 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 0fa6aa84c721c187cb4059599cabb328, NAME => 'TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:15,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 0fa6aa84c721c187cb4059599cabb328, NAME => 'TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:15,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fa6aa84c721c187cb4059599cabb328, NAME => 'TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fa6aa84c721c187cb4059599cabb328, NAME => 'TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:15,050 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 0fa6aa84c721c187cb4059599cabb328 in 1288ms, sequenceid=271, compaction requested=true 2024-12-09T10:41:15,050 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0fa6aa84c721c187cb4059599cabb328: 2024-12-09T10:41:15,051 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T10:41:15,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fa6aa84c721c187cb4059599cabb328:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T10:41:15,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:41:15,051 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T10:41:15,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fa6aa84c721c187cb4059599cabb328:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T10:41:15,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:41:15,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fa6aa84c721c187cb4059599cabb328:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T10:41:15,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T10:41:15,052 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94016 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T10:41:15,052 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1541): 0fa6aa84c721c187cb4059599cabb328/A is initiating minor compaction (all files) 2024-12-09T10:41:15,053 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fa6aa84c721c187cb4059599cabb328/A in TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. 2024-12-09T10:41:15,053 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/3a4dbebfc45f454e9bbcfcd797a67898, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/40f11f3552b542039dc14066a37d1a92, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/49529cf6fe984a33b2f0e31b6b07806f] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp, totalSize=91.8 K 2024-12-09T10:41:15,053 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. 2024-12-09T10:41:15,053 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. files: [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/3a4dbebfc45f454e9bbcfcd797a67898, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/40f11f3552b542039dc14066a37d1a92, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/49529cf6fe984a33b2f0e31b6b07806f] 2024-12-09T10:41:15,053 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37047 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T10:41:15,053 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1541): 0fa6aa84c721c187cb4059599cabb328/B is initiating minor compaction (all files) 2024-12-09T10:41:15,053 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fa6aa84c721c187cb4059599cabb328/B in TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. 2024-12-09T10:41:15,053 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3a4dbebfc45f454e9bbcfcd797a67898, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1733740870467 2024-12-09T10:41:15,053 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/19da3decfe974d69a605d61478e5dfa2, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/c4ecdc6ece1f4265902e0ff3c4bbbd61, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/ef18e1bcd1a5437890c7958856a67cd2] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp, totalSize=36.2 K 2024-12-09T10:41:15,054 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 40f11f3552b542039dc14066a37d1a92, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1733740871273 2024-12-09T10:41:15,054 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 19da3decfe974d69a605d61478e5dfa2, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1733740870467 2024-12-09T10:41:15,054 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 49529cf6fe984a33b2f0e31b6b07806f, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1733740873074 2024-12-09T10:41:15,054 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting c4ecdc6ece1f4265902e0ff3c4bbbd61, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1733740871273 2024-12-09T10:41:15,055 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting ef18e1bcd1a5437890c7958856a67cd2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1733740873074 2024-12-09T10:41:15,062 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0fa6aa84c721c187cb4059599cabb328] 2024-12-09T10:41:15,064 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fa6aa84c721c187cb4059599cabb328#B#compaction#162 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:41:15,064 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120976429ac2f84b4e5db8930f97cbdef96e_0fa6aa84c721c187cb4059599cabb328 store=[table=TestAcidGuarantees family=A region=0fa6aa84c721c187cb4059599cabb328] 2024-12-09T10:41:15,064 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/B/1e0a6269cead473692137011866cd376 is 50, key is test_row_0/B:col10/1733740873761/Put/seqid=0 2024-12-09T10:41:15,071 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120976429ac2f84b4e5db8930f97cbdef96e_0fa6aa84c721c187cb4059599cabb328, store=[table=TestAcidGuarantees family=A region=0fa6aa84c721c187cb4059599cabb328] 2024-12-09T10:41:15,071 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120976429ac2f84b4e5db8930f97cbdef96e_0fa6aa84c721c187cb4059599cabb328 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0fa6aa84c721c187cb4059599cabb328] 2024-12-09T10:41:15,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742018_1194 (size=12847) 2024-12-09T10:41:15,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742019_1195 (size=4469) 2024-12-09T10:41:15,089 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fa6aa84c721c187cb4059599cabb328#A#compaction#163 average throughput is 0.90 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:41:15,090 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/A/6b1c48de54e640aeaec09f3d920d2801 is 175, key is test_row_0/A:col10/1733740873761/Put/seqid=0 2024-12-09T10:41:15,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742020_1196 (size=31801) 2024-12-09T10:41:15,100 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/A/6b1c48de54e640aeaec09f3d920d2801 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/6b1c48de54e640aeaec09f3d920d2801 2024-12-09T10:41:15,106 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0fa6aa84c721c187cb4059599cabb328/A of 0fa6aa84c721c187cb4059599cabb328 into 6b1c48de54e640aeaec09f3d920d2801(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:41:15,106 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fa6aa84c721c187cb4059599cabb328: 2024-12-09T10:41:15,106 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., storeName=0fa6aa84c721c187cb4059599cabb328/A, priority=13, startTime=1733740875051; duration=0sec 2024-12-09T10:41:15,106 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T10:41:15,106 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fa6aa84c721c187cb4059599cabb328:A 2024-12-09T10:41:15,106 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T10:41:15,107 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37047 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T10:41:15,107 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1541): 0fa6aa84c721c187cb4059599cabb328/C is initiating minor compaction (all files) 2024-12-09T10:41:15,107 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fa6aa84c721c187cb4059599cabb328/C in TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. 2024-12-09T10:41:15,107 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/03eeddf98a654f10b0f01e80caf82eb5, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/553ba76ed01f4450b7c7bcbfc466dcaf, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/c56fd681b92a4616bdd927876145c32a] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp, totalSize=36.2 K 2024-12-09T10:41:15,108 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 03eeddf98a654f10b0f01e80caf82eb5, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1733740870467 2024-12-09T10:41:15,108 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 553ba76ed01f4450b7c7bcbfc466dcaf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1733740871273 2024-12-09T10:41:15,108 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting c56fd681b92a4616bdd927876145c32a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1733740873074 2024-12-09T10:41:15,116 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fa6aa84c721c187cb4059599cabb328#C#compaction#164 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:41:15,116 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/C/2ec0aea4ab3a4962ba33fd9ff364de21 is 50, key is test_row_0/C:col10/1733740873761/Put/seqid=0 2024-12-09T10:41:15,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742021_1197 (size=12847) 2024-12-09T10:41:15,129 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/C/2ec0aea4ab3a4962ba33fd9ff364de21 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/2ec0aea4ab3a4962ba33fd9ff364de21 2024-12-09T10:41:15,134 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0fa6aa84c721c187cb4059599cabb328/C of 0fa6aa84c721c187cb4059599cabb328 into 2ec0aea4ab3a4962ba33fd9ff364de21(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:41:15,134 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fa6aa84c721c187cb4059599cabb328: 2024-12-09T10:41:15,134 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., storeName=0fa6aa84c721c187cb4059599cabb328/C, priority=13, startTime=1733740875051; duration=0sec 2024-12-09T10:41:15,134 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:41:15,135 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fa6aa84c721c187cb4059599cabb328:C 2024-12-09T10:41:15,201 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-09T10:41:15,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. 2024-12-09T10:41:15,202 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing 0fa6aa84c721c187cb4059599cabb328 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-09T10:41:15,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fa6aa84c721c187cb4059599cabb328, store=A 2024-12-09T10:41:15,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:41:15,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fa6aa84c721c187cb4059599cabb328, store=B 2024-12-09T10:41:15,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:41:15,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fa6aa84c721c187cb4059599cabb328, store=C 2024-12-09T10:41:15,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:41:15,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412092d8f59ded98b47b4a02b655b0abee415_0fa6aa84c721c187cb4059599cabb328 is 50, key is test_row_0/A:col10/1733740873826/Put/seqid=0 2024-12-09T10:41:15,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742022_1198 (size=12454) 2024-12-09T10:41:15,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-09T10:41:15,482 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. as already flushing 2024-12-09T10:41:15,482 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/B/1e0a6269cead473692137011866cd376 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/1e0a6269cead473692137011866cd376 2024-12-09T10:41:15,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(8855): Flush requested on 0fa6aa84c721c187cb4059599cabb328 2024-12-09T10:41:15,489 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0fa6aa84c721c187cb4059599cabb328/B of 0fa6aa84c721c187cb4059599cabb328 into 1e0a6269cead473692137011866cd376(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:41:15,489 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fa6aa84c721c187cb4059599cabb328: 2024-12-09T10:41:15,489 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., storeName=0fa6aa84c721c187cb4059599cabb328/B, priority=13, startTime=1733740875051; duration=0sec 2024-12-09T10:41:15,489 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:41:15,489 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fa6aa84c721c187cb4059599cabb328:B 2024-12-09T10:41:15,509 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:15,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54262 deadline: 1733740935506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:15,509 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:15,509 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:15,510 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:15,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:15,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54216 deadline: 1733740935509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:15,514 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:15,514 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:15,515 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:15,622 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:15,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54262 deadline: 1733740935620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:15,622 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:15,622 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:15,622 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:15,622 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:15,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54216 deadline: 1733740935620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:15,623 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:15,623 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:15,623 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:15,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:15,636 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412092d8f59ded98b47b4a02b655b0abee415_0fa6aa84c721c187cb4059599cabb328 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412092d8f59ded98b47b4a02b655b0abee415_0fa6aa84c721c187cb4059599cabb328 2024-12-09T10:41:15,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/A/f807342bd7df4170b93ac61f55053f2d, store: [table=TestAcidGuarantees family=A region=0fa6aa84c721c187cb4059599cabb328] 2024-12-09T10:41:15,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/A/f807342bd7df4170b93ac61f55053f2d is 175, key is test_row_0/A:col10/1733740873826/Put/seqid=0 2024-12-09T10:41:15,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742023_1199 (size=31255) 2024-12-09T10:41:15,834 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:15,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54262 deadline: 1733740935829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:15,834 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:15,834 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:15,834 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:15,834 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:15,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54216 deadline: 1733740935829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:15,835 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:15,835 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:15,835 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:15,982 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:15,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54232 deadline: 1733740935980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:15,983 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 7ea43b39fc24,36101,1733740827678 for 'test_row_1' in 0fa6aa84c721c187cb4059599cabb328 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2149 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T10:41:15,984 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:15,984 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:15,984 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:16,044 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=291, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/A/f807342bd7df4170b93ac61f55053f2d 2024-12-09T10:41:16,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/B/8fee427cf86b4c70b796ba918f2b30c0 is 50, key is test_row_0/B:col10/1733740873826/Put/seqid=0 2024-12-09T10:41:16,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742024_1200 (size=12301) 2024-12-09T10:41:16,144 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:16,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54262 deadline: 1733740936139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:16,145 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:16,145 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:16,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54216 deadline: 1733740936139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:16,145 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:16,145 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:16,145 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:16,145 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:16,145 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:16,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-09T10:41:16,459 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/B/8fee427cf86b4c70b796ba918f2b30c0 2024-12-09T10:41:16,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/C/47cf06ed32ee4f35acd7479ab214efe2 is 50, key is test_row_0/C:col10/1733740873826/Put/seqid=0 2024-12-09T10:41:16,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742025_1201 (size=12301) 2024-12-09T10:41:16,514 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/C/47cf06ed32ee4f35acd7479ab214efe2 2024-12-09T10:41:16,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/A/f807342bd7df4170b93ac61f55053f2d as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/f807342bd7df4170b93ac61f55053f2d 2024-12-09T10:41:16,532 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/f807342bd7df4170b93ac61f55053f2d, entries=150, sequenceid=291, filesize=30.5 K 2024-12-09T10:41:16,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/B/8fee427cf86b4c70b796ba918f2b30c0 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/8fee427cf86b4c70b796ba918f2b30c0 2024-12-09T10:41:16,543 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/8fee427cf86b4c70b796ba918f2b30c0, entries=150, sequenceid=291, filesize=12.0 K 2024-12-09T10:41:16,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/C/47cf06ed32ee4f35acd7479ab214efe2 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/47cf06ed32ee4f35acd7479ab214efe2 2024-12-09T10:41:16,551 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/47cf06ed32ee4f35acd7479ab214efe2, entries=150, sequenceid=291, filesize=12.0 K 2024-12-09T10:41:16,552 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 0fa6aa84c721c187cb4059599cabb328 in 1350ms, sequenceid=291, compaction requested=false 2024-12-09T10:41:16,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for 0fa6aa84c721c187cb4059599cabb328: 2024-12-09T10:41:16,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. 2024-12-09T10:41:16,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-12-09T10:41:16,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-12-09T10:41:16,556 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-12-09T10:41:16,556 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2710 sec 2024-12-09T10:41:16,559 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 2.2770 sec 2024-12-09T10:41:16,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(8855): Flush requested on 0fa6aa84c721c187cb4059599cabb328 2024-12-09T10:41:16,652 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0fa6aa84c721c187cb4059599cabb328 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-09T10:41:16,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fa6aa84c721c187cb4059599cabb328, store=A 2024-12-09T10:41:16,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:41:16,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fa6aa84c721c187cb4059599cabb328, store=B 2024-12-09T10:41:16,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:41:16,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fa6aa84c721c187cb4059599cabb328, store=C 2024-12-09T10:41:16,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:41:16,661 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120966a0e3cea85443878a1b0e56a0978004_0fa6aa84c721c187cb4059599cabb328 is 50, key is test_row_0/A:col10/1733740876651/Put/seqid=0 2024-12-09T10:41:16,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742026_1202 (size=14994) 2024-12-09T10:41:16,743 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:16,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54262 deadline: 1733740936741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:16,743 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:16,743 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:16,744 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:16,744 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:16,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54216 deadline: 1733740936743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:16,744 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:16,744 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:16,744 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:16,853 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:16,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54262 deadline: 1733740936849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:16,853 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:16,853 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:16,853 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:16,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54216 deadline: 1733740936849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:16,853 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:16,854 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:16,854 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:16,854 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:17,060 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:17,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54262 deadline: 1733740937059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:17,061 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:17,061 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:17,061 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:17,061 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:17,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54216 deadline: 1733740937059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:17,061 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:17,061 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:17,061 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:17,070 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:17,075 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120966a0e3cea85443878a1b0e56a0978004_0fa6aa84c721c187cb4059599cabb328 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120966a0e3cea85443878a1b0e56a0978004_0fa6aa84c721c187cb4059599cabb328 2024-12-09T10:41:17,076 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/A/d0bfa67bfa2747309f337fef8c9fa553, store: [table=TestAcidGuarantees family=A region=0fa6aa84c721c187cb4059599cabb328] 2024-12-09T10:41:17,076 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/A/d0bfa67bfa2747309f337fef8c9fa553 is 175, key is test_row_0/A:col10/1733740876651/Put/seqid=0 2024-12-09T10:41:17,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742027_1203 (size=39949) 2024-12-09T10:41:17,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:17,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54262 deadline: 1733740937369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:17,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:17,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:17,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:17,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:17,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54216 deadline: 1733740937369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:17,373 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:17,373 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:17,373 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:17,488 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=307, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/A/d0bfa67bfa2747309f337fef8c9fa553 2024-12-09T10:41:17,496 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/B/efa1279dd03d4046a1437c417efc442d is 50, key is test_row_0/B:col10/1733740876651/Put/seqid=0 2024-12-09T10:41:17,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742028_1204 (size=12301) 2024-12-09T10:41:17,880 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:17,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54262 deadline: 1733740937879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:17,881 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:17,881 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:17,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54216 deadline: 1733740937879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:17,881 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:17,881 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:17,881 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:17,881 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:17,881 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:17,900 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=307 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/B/efa1279dd03d4046a1437c417efc442d 2024-12-09T10:41:17,908 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/C/5a68f4b86db14504a06085f9546fba96 is 50, key is test_row_0/C:col10/1733740876651/Put/seqid=0 2024-12-09T10:41:17,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742029_1205 (size=12301) 2024-12-09T10:41:18,002 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:18,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54232 deadline: 1733740937999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:18,002 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 7ea43b39fc24,36101,1733740827678 for 'test_row_1' in 0fa6aa84c721c187cb4059599cabb328 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T10:41:18,003 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:18,003 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:18,003 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:18,314 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=307 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/C/5a68f4b86db14504a06085f9546fba96 2024-12-09T10:41:18,319 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/A/d0bfa67bfa2747309f337fef8c9fa553 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/d0bfa67bfa2747309f337fef8c9fa553 2024-12-09T10:41:18,325 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/d0bfa67bfa2747309f337fef8c9fa553, entries=200, sequenceid=307, filesize=39.0 K 2024-12-09T10:41:18,326 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/B/efa1279dd03d4046a1437c417efc442d as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/efa1279dd03d4046a1437c417efc442d 2024-12-09T10:41:18,331 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/efa1279dd03d4046a1437c417efc442d, entries=150, sequenceid=307, filesize=12.0 K 2024-12-09T10:41:18,332 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/C/5a68f4b86db14504a06085f9546fba96 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/5a68f4b86db14504a06085f9546fba96 2024-12-09T10:41:18,336 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/5a68f4b86db14504a06085f9546fba96, entries=150, sequenceid=307, filesize=12.0 K 2024-12-09T10:41:18,337 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 0fa6aa84c721c187cb4059599cabb328 in 1685ms, sequenceid=307, compaction requested=true 2024-12-09T10:41:18,337 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0fa6aa84c721c187cb4059599cabb328: 2024-12-09T10:41:18,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fa6aa84c721c187cb4059599cabb328:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T10:41:18,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:41:18,338 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T10:41:18,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fa6aa84c721c187cb4059599cabb328:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T10:41:18,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:41:18,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fa6aa84c721c187cb4059599cabb328:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T10:41:18,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T10:41:18,338 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T10:41:18,339 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103005 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T10:41:18,339 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T10:41:18,339 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1541): 0fa6aa84c721c187cb4059599cabb328/A is initiating minor compaction (all files) 2024-12-09T10:41:18,339 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1541): 0fa6aa84c721c187cb4059599cabb328/B is initiating minor compaction (all files) 2024-12-09T10:41:18,339 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fa6aa84c721c187cb4059599cabb328/A in TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. 2024-12-09T10:41:18,339 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fa6aa84c721c187cb4059599cabb328/B in TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. 2024-12-09T10:41:18,339 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/6b1c48de54e640aeaec09f3d920d2801, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/f807342bd7df4170b93ac61f55053f2d, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/d0bfa67bfa2747309f337fef8c9fa553] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp, totalSize=100.6 K 2024-12-09T10:41:18,339 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. 2024-12-09T10:41:18,339 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/1e0a6269cead473692137011866cd376, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/8fee427cf86b4c70b796ba918f2b30c0, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/efa1279dd03d4046a1437c417efc442d] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp, totalSize=36.6 K 2024-12-09T10:41:18,339 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. files: [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/6b1c48de54e640aeaec09f3d920d2801, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/f807342bd7df4170b93ac61f55053f2d, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/d0bfa67bfa2747309f337fef8c9fa553] 2024-12-09T10:41:18,339 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b1c48de54e640aeaec09f3d920d2801, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1733740873074 2024-12-09T10:41:18,339 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e0a6269cead473692137011866cd376, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1733740873074 2024-12-09T10:41:18,340 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting f807342bd7df4170b93ac61f55053f2d, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1733740873819 2024-12-09T10:41:18,340 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 8fee427cf86b4c70b796ba918f2b30c0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1733740873819 2024-12-09T10:41:18,340 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting d0bfa67bfa2747309f337fef8c9fa553, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1733740875491 2024-12-09T10:41:18,340 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting efa1279dd03d4046a1437c417efc442d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1733740875491 2024-12-09T10:41:18,348 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fa6aa84c721c187cb4059599cabb328#B#compaction#171 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:41:18,349 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/B/0ec8ff9bb1fc4d05b34777572b2e677f is 50, key is test_row_0/B:col10/1733740876651/Put/seqid=0 2024-12-09T10:41:18,351 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0fa6aa84c721c187cb4059599cabb328] 2024-12-09T10:41:18,354 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412095496b6e9ac0f48c5ab7e497011296716_0fa6aa84c721c187cb4059599cabb328 store=[table=TestAcidGuarantees family=A region=0fa6aa84c721c187cb4059599cabb328] 2024-12-09T10:41:18,355 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412095496b6e9ac0f48c5ab7e497011296716_0fa6aa84c721c187cb4059599cabb328, store=[table=TestAcidGuarantees family=A region=0fa6aa84c721c187cb4059599cabb328] 2024-12-09T10:41:18,356 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412095496b6e9ac0f48c5ab7e497011296716_0fa6aa84c721c187cb4059599cabb328 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0fa6aa84c721c187cb4059599cabb328] 2024-12-09T10:41:18,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742030_1206 (size=12949) 2024-12-09T10:41:18,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742031_1207 (size=4469) 2024-12-09T10:41:18,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-09T10:41:18,440 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T10:41:18,440 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-09T10:41:18,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-12-09T10:41:18,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-09T10:41:18,442 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T10:41:18,443 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T10:41:18,443 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T10:41:18,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-09T10:41:18,596 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-09T10:41:18,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. 2024-12-09T10:41:18,596 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing 0fa6aa84c721c187cb4059599cabb328 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-09T10:41:18,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fa6aa84c721c187cb4059599cabb328, store=A 2024-12-09T10:41:18,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:41:18,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fa6aa84c721c187cb4059599cabb328, store=B 2024-12-09T10:41:18,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:41:18,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fa6aa84c721c187cb4059599cabb328, store=C 2024-12-09T10:41:18,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:41:18,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120986f6747554744423b83c3157ed10afa4_0fa6aa84c721c187cb4059599cabb328 is 50, key is test_row_0/A:col10/1733740876726/Put/seqid=0 2024-12-09T10:41:18,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742032_1208 (size=12454) 2024-12-09T10:41:18,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-09T10:41:18,761 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fa6aa84c721c187cb4059599cabb328#A#compaction#172 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:41:18,761 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/A/b4256eab5ebc4ca2ae7b1900a961b509 is 175, key is test_row_0/A:col10/1733740876651/Put/seqid=0 2024-12-09T10:41:18,762 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/B/0ec8ff9bb1fc4d05b34777572b2e677f as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/0ec8ff9bb1fc4d05b34777572b2e677f 2024-12-09T10:41:18,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742033_1209 (size=31903) 2024-12-09T10:41:18,773 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0fa6aa84c721c187cb4059599cabb328/B of 0fa6aa84c721c187cb4059599cabb328 into 0ec8ff9bb1fc4d05b34777572b2e677f(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:41:18,773 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fa6aa84c721c187cb4059599cabb328: 2024-12-09T10:41:18,773 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., storeName=0fa6aa84c721c187cb4059599cabb328/B, priority=13, startTime=1733740878338; duration=0sec 2024-12-09T10:41:18,773 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T10:41:18,773 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fa6aa84c721c187cb4059599cabb328:B 2024-12-09T10:41:18,773 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T10:41:18,775 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T10:41:18,775 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1541): 0fa6aa84c721c187cb4059599cabb328/C is initiating minor compaction (all files) 2024-12-09T10:41:18,775 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fa6aa84c721c187cb4059599cabb328/C in TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. 2024-12-09T10:41:18,775 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/2ec0aea4ab3a4962ba33fd9ff364de21, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/47cf06ed32ee4f35acd7479ab214efe2, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/5a68f4b86db14504a06085f9546fba96] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp, totalSize=36.6 K 2024-12-09T10:41:18,775 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ec0aea4ab3a4962ba33fd9ff364de21, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1733740873074 2024-12-09T10:41:18,776 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 47cf06ed32ee4f35acd7479ab214efe2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1733740873819 2024-12-09T10:41:18,776 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a68f4b86db14504a06085f9546fba96, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1733740875491 2024-12-09T10:41:18,786 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fa6aa84c721c187cb4059599cabb328#C#compaction#174 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:41:18,787 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/C/7d16e799c47b43dc91349e9cc62324a3 is 50, key is test_row_0/C:col10/1733740876651/Put/seqid=0 2024-12-09T10:41:18,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742034_1210 (size=12949) 2024-12-09T10:41:18,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(8855): Flush requested on 0fa6aa84c721c187cb4059599cabb328 2024-12-09T10:41:18,890 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. as already flushing 2024-12-09T10:41:18,924 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:18,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54216 deadline: 1733740938919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:18,924 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:18,924 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:18,924 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:18,925 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:18,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54262 deadline: 1733740938924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:18,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:18,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:18,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:19,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:19,013 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120986f6747554744423b83c3157ed10afa4_0fa6aa84c721c187cb4059599cabb328 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120986f6747554744423b83c3157ed10afa4_0fa6aa84c721c187cb4059599cabb328 2024-12-09T10:41:19,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/A/ace48c0b600a4403af32be9a228cc732, store: [table=TestAcidGuarantees family=A region=0fa6aa84c721c187cb4059599cabb328] 2024-12-09T10:41:19,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/A/ace48c0b600a4403af32be9a228cc732 is 175, key is test_row_0/A:col10/1733740876726/Put/seqid=0 2024-12-09T10:41:19,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742035_1211 (size=31255) 2024-12-09T10:41:19,032 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:19,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54216 deadline: 1733740939030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:19,033 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:19,033 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:19,033 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:19,033 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:19,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54262 deadline: 1733740939030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:19,033 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:19,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:19,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:19,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-09T10:41:19,174 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/A/b4256eab5ebc4ca2ae7b1900a961b509 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/b4256eab5ebc4ca2ae7b1900a961b509 2024-12-09T10:41:19,180 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0fa6aa84c721c187cb4059599cabb328/A of 0fa6aa84c721c187cb4059599cabb328 into b4256eab5ebc4ca2ae7b1900a961b509(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:41:19,180 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fa6aa84c721c187cb4059599cabb328: 2024-12-09T10:41:19,180 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., storeName=0fa6aa84c721c187cb4059599cabb328/A, priority=13, startTime=1733740878337; duration=0sec 2024-12-09T10:41:19,180 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:41:19,180 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fa6aa84c721c187cb4059599cabb328:A 2024-12-09T10:41:19,199 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/C/7d16e799c47b43dc91349e9cc62324a3 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/7d16e799c47b43dc91349e9cc62324a3 2024-12-09T10:41:19,204 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0fa6aa84c721c187cb4059599cabb328/C of 0fa6aa84c721c187cb4059599cabb328 into 7d16e799c47b43dc91349e9cc62324a3(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:41:19,204 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fa6aa84c721c187cb4059599cabb328: 2024-12-09T10:41:19,204 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., storeName=0fa6aa84c721c187cb4059599cabb328/C, priority=13, startTime=1733740878338; duration=0sec 2024-12-09T10:41:19,204 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:41:19,204 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fa6aa84c721c187cb4059599cabb328:C 2024-12-09T10:41:19,243 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:19,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54216 deadline: 1733740939240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:19,243 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:19,243 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:19,243 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:19,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54262 deadline: 1733740939240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:19,243 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:19,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:19,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:19,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:19,419 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=324, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/A/ace48c0b600a4403af32be9a228cc732 2024-12-09T10:41:19,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/B/6087f5b8160d49d9aba0aa435d257b4f is 50, key is test_row_0/B:col10/1733740876726/Put/seqid=0 2024-12-09T10:41:19,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742036_1212 (size=12301) 2024-12-09T10:41:19,551 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:19,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54216 deadline: 1733740939549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:19,551 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:19,551 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:19,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54262 deadline: 1733740939549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:19,551 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:19,551 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:19,551 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:19,552 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:19,552 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:19,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-09T10:41:19,833 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/B/6087f5b8160d49d9aba0aa435d257b4f 2024-12-09T10:41:19,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/C/b98a7d52903c416a80a520be89f14a08 is 50, key is test_row_0/C:col10/1733740876726/Put/seqid=0 2024-12-09T10:41:19,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742037_1213 (size=12301) 2024-12-09T10:41:19,847 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/C/b98a7d52903c416a80a520be89f14a08 2024-12-09T10:41:19,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/A/ace48c0b600a4403af32be9a228cc732 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/ace48c0b600a4403af32be9a228cc732 2024-12-09T10:41:19,857 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/ace48c0b600a4403af32be9a228cc732, entries=150, sequenceid=324, filesize=30.5 K 2024-12-09T10:41:19,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/B/6087f5b8160d49d9aba0aa435d257b4f as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/6087f5b8160d49d9aba0aa435d257b4f 2024-12-09T10:41:19,862 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/6087f5b8160d49d9aba0aa435d257b4f, entries=150, sequenceid=324, filesize=12.0 K 2024-12-09T10:41:19,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/C/b98a7d52903c416a80a520be89f14a08 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/b98a7d52903c416a80a520be89f14a08 2024-12-09T10:41:19,867 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/b98a7d52903c416a80a520be89f14a08, entries=150, sequenceid=324, filesize=12.0 K 2024-12-09T10:41:19,868 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 0fa6aa84c721c187cb4059599cabb328 in 1272ms, sequenceid=324, compaction requested=false 2024-12-09T10:41:19,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for 0fa6aa84c721c187cb4059599cabb328: 2024-12-09T10:41:19,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. 2024-12-09T10:41:19,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-12-09T10:41:19,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-12-09T10:41:19,872 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-12-09T10:41:19,872 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4270 sec 2024-12-09T10:41:19,874 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 1.4320 sec 2024-12-09T10:41:20,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(8855): Flush requested on 0fa6aa84c721c187cb4059599cabb328 2024-12-09T10:41:20,066 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0fa6aa84c721c187cb4059599cabb328 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-09T10:41:20,067 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fa6aa84c721c187cb4059599cabb328, store=A 2024-12-09T10:41:20,067 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:41:20,067 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fa6aa84c721c187cb4059599cabb328, store=B 2024-12-09T10:41:20,067 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:41:20,067 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fa6aa84c721c187cb4059599cabb328, store=C 2024-12-09T10:41:20,067 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:41:20,075 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209c56833f921ee4edbbbae096026d02cdc_0fa6aa84c721c187cb4059599cabb328 is 50, key is test_row_0/A:col10/1733740878923/Put/seqid=0 2024-12-09T10:41:20,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742038_1214 (size=17534) 2024-12-09T10:41:20,096 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:20,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54216 deadline: 1733740940092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:20,096 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:20,096 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:20,096 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:20,096 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:20,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54262 deadline: 1733740940095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:20,097 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:20,097 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:20,097 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:20,183 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:20,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54272 deadline: 1733740940180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:20,184 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 7ea43b39fc24,36101,1733740827678 for 'test_row_2' in 0fa6aa84c721c187cb4059599cabb328 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18346 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T10:41:20,185 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:20,185 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:20,185 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:20,202 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:20,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54258 deadline: 1733740940200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:20,203 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:20,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54216 deadline: 1733740940200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:20,203 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:20,203 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:20,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54262 deadline: 1733740940200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:20,203 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:20,203 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:20,203 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 7ea43b39fc24,36101,1733740827678 for 'test_row_1' in 0fa6aa84c721c187cb4059599cabb328 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18367 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T10:41:20,203 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:20,203 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:20,203 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:20,203 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:20,203 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:20,204 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:20,411 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:20,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54216 deadline: 1733740940410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:20,411 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:20,411 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:20,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54262 deadline: 1733740940410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:20,411 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:20,411 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:20,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:20,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:20,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:20,482 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:20,487 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209c56833f921ee4edbbbae096026d02cdc_0fa6aa84c721c187cb4059599cabb328 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209c56833f921ee4edbbbae096026d02cdc_0fa6aa84c721c187cb4059599cabb328 2024-12-09T10:41:20,488 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/A/b4e47bcb3a544cb1ab0c67aec1ee7df0, store: [table=TestAcidGuarantees family=A region=0fa6aa84c721c187cb4059599cabb328] 2024-12-09T10:41:20,488 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/A/b4e47bcb3a544cb1ab0c67aec1ee7df0 is 175, key is test_row_0/A:col10/1733740878923/Put/seqid=0 2024-12-09T10:41:20,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742039_1215 (size=48639) 2024-12-09T10:41:20,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-09T10:41:20,610 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T10:41:20,610 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-09T10:41:20,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-12-09T10:41:20,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-09T10:41:20,612 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T10:41:20,613 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T10:41:20,613 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T10:41:20,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-09T10:41:20,723 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:20,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54216 deadline: 1733740940720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:20,724 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:20,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54262 deadline: 1733740940720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:20,724 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:20,724 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:20,724 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:20,724 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:20,724 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:20,724 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:20,766 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-09T10:41:20,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. 2024-12-09T10:41:20,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. as already flushing 2024-12-09T10:41:20,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. 2024-12-09T10:41:20,766 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 0fa6aa84c721c187cb4059599cabb328, NAME => 'TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:20,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 0fa6aa84c721c187cb4059599cabb328, NAME => 'TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:20,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fa6aa84c721c187cb4059599cabb328, NAME => 'TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fa6aa84c721c187cb4059599cabb328, NAME => 'TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:20,894 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=344, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/A/b4e47bcb3a544cb1ab0c67aec1ee7df0 2024-12-09T10:41:20,905 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/B/445420969aa342859444af5c63e864c2 is 50, key is test_row_0/B:col10/1733740878923/Put/seqid=0 2024-12-09T10:41:20,919 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-09T10:41:20,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. 2024-12-09T10:41:20,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. as already flushing 2024-12-09T10:41:20,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. 2024-12-09T10:41:20,920 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 0fa6aa84c721c187cb4059599cabb328, NAME => 'TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:20,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 0fa6aa84c721c187cb4059599cabb328, NAME => 'TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:20,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742040_1216 (size=12301) 2024-12-09T10:41:20,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fa6aa84c721c187cb4059599cabb328, NAME => 'TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fa6aa84c721c187cb4059599cabb328, NAME => 'TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:20,923 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=344 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/B/445420969aa342859444af5c63e864c2 2024-12-09T10:41:20,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-09T10:41:20,937 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/C/dd9629d51cc44451bca486924e80a08a is 50, key is test_row_0/C:col10/1733740878923/Put/seqid=0 2024-12-09T10:41:20,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742041_1217 (size=12301) 2024-12-09T10:41:20,947 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=344 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/C/dd9629d51cc44451bca486924e80a08a 2024-12-09T10:41:20,953 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/A/b4e47bcb3a544cb1ab0c67aec1ee7df0 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/b4e47bcb3a544cb1ab0c67aec1ee7df0 2024-12-09T10:41:20,962 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/b4e47bcb3a544cb1ab0c67aec1ee7df0, entries=250, sequenceid=344, filesize=47.5 K 2024-12-09T10:41:20,964 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/B/445420969aa342859444af5c63e864c2 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/445420969aa342859444af5c63e864c2 2024-12-09T10:41:20,970 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/445420969aa342859444af5c63e864c2, entries=150, sequenceid=344, filesize=12.0 K 2024-12-09T10:41:20,971 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/C/dd9629d51cc44451bca486924e80a08a as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/dd9629d51cc44451bca486924e80a08a 2024-12-09T10:41:20,978 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/dd9629d51cc44451bca486924e80a08a, entries=150, sequenceid=344, filesize=12.0 K 2024-12-09T10:41:20,979 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 0fa6aa84c721c187cb4059599cabb328 in 913ms, sequenceid=344, compaction requested=true 2024-12-09T10:41:20,980 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0fa6aa84c721c187cb4059599cabb328: 2024-12-09T10:41:20,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fa6aa84c721c187cb4059599cabb328:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T10:41:20,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:41:20,980 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T10:41:20,980 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T10:41:20,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fa6aa84c721c187cb4059599cabb328:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T10:41:20,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:41:20,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fa6aa84c721c187cb4059599cabb328:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T10:41:20,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T10:41:20,984 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T10:41:20,984 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1541): 0fa6aa84c721c187cb4059599cabb328/B is initiating minor compaction (all files) 2024-12-09T10:41:20,984 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fa6aa84c721c187cb4059599cabb328/B in TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. 2024-12-09T10:41:20,984 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/0ec8ff9bb1fc4d05b34777572b2e677f, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/6087f5b8160d49d9aba0aa435d257b4f, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/445420969aa342859444af5c63e864c2] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp, totalSize=36.7 K 2024-12-09T10:41:20,984 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111797 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T10:41:20,985 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1541): 0fa6aa84c721c187cb4059599cabb328/A is initiating minor compaction (all files) 2024-12-09T10:41:20,985 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 0ec8ff9bb1fc4d05b34777572b2e677f, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1733740875491 2024-12-09T10:41:20,985 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fa6aa84c721c187cb4059599cabb328/A in TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. 2024-12-09T10:41:20,985 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/b4256eab5ebc4ca2ae7b1900a961b509, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/ace48c0b600a4403af32be9a228cc732, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/b4e47bcb3a544cb1ab0c67aec1ee7df0] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp, totalSize=109.2 K 2024-12-09T10:41:20,985 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. 2024-12-09T10:41:20,985 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. files: [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/b4256eab5ebc4ca2ae7b1900a961b509, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/ace48c0b600a4403af32be9a228cc732, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/b4e47bcb3a544cb1ab0c67aec1ee7df0] 2024-12-09T10:41:20,985 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 6087f5b8160d49d9aba0aa435d257b4f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1733740876726 2024-12-09T10:41:20,985 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting b4256eab5ebc4ca2ae7b1900a961b509, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1733740875491 2024-12-09T10:41:20,987 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting ace48c0b600a4403af32be9a228cc732, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1733740876726 2024-12-09T10:41:20,987 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 445420969aa342859444af5c63e864c2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1733740878923 2024-12-09T10:41:20,988 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting b4e47bcb3a544cb1ab0c67aec1ee7df0, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1733740878907 2024-12-09T10:41:21,002 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0fa6aa84c721c187cb4059599cabb328] 2024-12-09T10:41:21,004 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fa6aa84c721c187cb4059599cabb328#B#compaction#180 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:41:21,006 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/B/4de14da0877545ce8ffb2bb45e207b62 is 50, key is test_row_0/B:col10/1733740878923/Put/seqid=0 2024-12-09T10:41:21,008 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241209e3793464b7d44e2ba359de296b104569_0fa6aa84c721c187cb4059599cabb328 store=[table=TestAcidGuarantees family=A region=0fa6aa84c721c187cb4059599cabb328] 2024-12-09T10:41:21,010 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241209e3793464b7d44e2ba359de296b104569_0fa6aa84c721c187cb4059599cabb328, store=[table=TestAcidGuarantees family=A region=0fa6aa84c721c187cb4059599cabb328] 2024-12-09T10:41:21,010 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209e3793464b7d44e2ba359de296b104569_0fa6aa84c721c187cb4059599cabb328 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0fa6aa84c721c187cb4059599cabb328] 2024-12-09T10:41:21,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742042_1218 (size=13051) 2024-12-09T10:41:21,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742043_1219 (size=4469) 2024-12-09T10:41:21,035 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fa6aa84c721c187cb4059599cabb328#A#compaction#181 average throughput is 0.74 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:41:21,036 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/A/53897573d28b48db984f709c6f269cfd is 175, key is test_row_0/A:col10/1733740878923/Put/seqid=0 2024-12-09T10:41:21,041 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/B/4de14da0877545ce8ffb2bb45e207b62 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/4de14da0877545ce8ffb2bb45e207b62 2024-12-09T10:41:21,046 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0fa6aa84c721c187cb4059599cabb328/B of 0fa6aa84c721c187cb4059599cabb328 into 4de14da0877545ce8ffb2bb45e207b62(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:41:21,046 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fa6aa84c721c187cb4059599cabb328: 2024-12-09T10:41:21,047 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., storeName=0fa6aa84c721c187cb4059599cabb328/B, priority=13, startTime=1733740880980; duration=0sec 2024-12-09T10:41:21,047 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T10:41:21,047 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fa6aa84c721c187cb4059599cabb328:B 2024-12-09T10:41:21,047 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T10:41:21,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742044_1220 (size=32005) 2024-12-09T10:41:21,052 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T10:41:21,052 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1541): 0fa6aa84c721c187cb4059599cabb328/C is initiating minor compaction (all files) 2024-12-09T10:41:21,052 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fa6aa84c721c187cb4059599cabb328/C in TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. 2024-12-09T10:41:21,053 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/7d16e799c47b43dc91349e9cc62324a3, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/b98a7d52903c416a80a520be89f14a08, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/dd9629d51cc44451bca486924e80a08a] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp, totalSize=36.7 K 2024-12-09T10:41:21,053 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d16e799c47b43dc91349e9cc62324a3, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1733740875491 2024-12-09T10:41:21,059 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting b98a7d52903c416a80a520be89f14a08, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1733740876726 2024-12-09T10:41:21,060 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting dd9629d51cc44451bca486924e80a08a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1733740878923 2024-12-09T10:41:21,074 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/A/53897573d28b48db984f709c6f269cfd as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/53897573d28b48db984f709c6f269cfd 2024-12-09T10:41:21,074 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-09T10:41:21,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. 2024-12-09T10:41:21,074 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing 0fa6aa84c721c187cb4059599cabb328 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-09T10:41:21,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fa6aa84c721c187cb4059599cabb328, store=A 2024-12-09T10:41:21,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:41:21,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fa6aa84c721c187cb4059599cabb328, store=B 2024-12-09T10:41:21,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:41:21,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fa6aa84c721c187cb4059599cabb328, store=C 2024-12-09T10:41:21,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:41:21,080 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fa6aa84c721c187cb4059599cabb328#C#compaction#182 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:41:21,080 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/C/7111523abe794d46b92959727e070e09 is 50, key is test_row_0/C:col10/1733740878923/Put/seqid=0 2024-12-09T10:41:21,089 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0fa6aa84c721c187cb4059599cabb328/A of 0fa6aa84c721c187cb4059599cabb328 into 53897573d28b48db984f709c6f269cfd(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:41:21,089 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fa6aa84c721c187cb4059599cabb328: 2024-12-09T10:41:21,089 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., storeName=0fa6aa84c721c187cb4059599cabb328/A, priority=13, startTime=1733740880980; duration=0sec 2024-12-09T10:41:21,089 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:41:21,089 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fa6aa84c721c187cb4059599cabb328:A 2024-12-09T10:41:21,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412092d4f5d6929404a698659ea9fef0b066d_0fa6aa84c721c187cb4059599cabb328 is 50, key is test_row_0/A:col10/1733740880086/Put/seqid=0 2024-12-09T10:41:21,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742045_1221 (size=13051) 2024-12-09T10:41:21,149 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/C/7111523abe794d46b92959727e070e09 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/7111523abe794d46b92959727e070e09 2024-12-09T10:41:21,157 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0fa6aa84c721c187cb4059599cabb328/C of 0fa6aa84c721c187cb4059599cabb328 into 7111523abe794d46b92959727e070e09(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:41:21,157 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fa6aa84c721c187cb4059599cabb328: 2024-12-09T10:41:21,157 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., storeName=0fa6aa84c721c187cb4059599cabb328/C, priority=13, startTime=1733740880980; duration=0sec 2024-12-09T10:41:21,157 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:41:21,157 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fa6aa84c721c187cb4059599cabb328:C 2024-12-09T10:41:21,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742046_1222 (size=12454) 2024-12-09T10:41:21,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:21,176 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412092d4f5d6929404a698659ea9fef0b066d_0fa6aa84c721c187cb4059599cabb328 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412092d4f5d6929404a698659ea9fef0b066d_0fa6aa84c721c187cb4059599cabb328 2024-12-09T10:41:21,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/A/7ab10fca724f4f6680f3698ad347b254, store: [table=TestAcidGuarantees family=A region=0fa6aa84c721c187cb4059599cabb328] 2024-12-09T10:41:21,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/A/7ab10fca724f4f6680f3698ad347b254 is 175, key is test_row_0/A:col10/1733740880086/Put/seqid=0 2024-12-09T10:41:21,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742047_1223 (size=31255) 2024-12-09T10:41:21,189 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=360, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/A/7ab10fca724f4f6680f3698ad347b254 2024-12-09T10:41:21,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/B/cd5a41d5e4334ae8a558da3feea275d2 is 50, key is test_row_0/B:col10/1733740880086/Put/seqid=0 2024-12-09T10:41:21,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742048_1224 (size=12301) 2024-12-09T10:41:21,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(8855): Flush requested on 0fa6aa84c721c187cb4059599cabb328 2024-12-09T10:41:21,231 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. as already flushing 2024-12-09T10:41:21,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-09T10:41:21,262 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:21,262 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:21,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54262 deadline: 1733740941258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:21,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54216 deadline: 1733740941258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:21,263 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:21,263 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:21,263 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:21,263 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:21,263 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:21,263 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:21,370 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:21,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54216 deadline: 1733740941369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:21,370 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:21,370 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:21,370 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:21,371 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:21,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54262 deadline: 1733740941370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:21,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:21,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:21,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:21,583 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:21,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54216 deadline: 1733740941579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:21,583 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:21,583 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:21,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54262 deadline: 1733740941579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:21,583 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:21,583 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:21,583 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:21,583 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:21,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:21,605 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/B/cd5a41d5e4334ae8a558da3feea275d2 2024-12-09T10:41:21,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/C/237ac8d1a5a04bc9a600932891c6d48b is 50, key is test_row_0/C:col10/1733740880086/Put/seqid=0 2024-12-09T10:41:21,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742049_1225 (size=12301) 2024-12-09T10:41:21,689 INFO [Thread-658 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-658. 2024-12-09T10:41:21,689 DEBUG [Thread-658 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T10:41:21,689 DEBUG [Thread-658 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:41:21,689 DEBUG [Thread-658 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:41:21,689 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T10:41:21,693 INFO [Thread-662 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-662. 2024-12-09T10:41:21,693 DEBUG [Thread-662 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T10:41:21,693 DEBUG [Thread-662 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:41:21,693 DEBUG [Thread-662 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:41:21,693 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T10:41:21,694 INFO [Thread-656 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-656. 2024-12-09T10:41:21,694 DEBUG [Thread-656 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T10:41:21,694 DEBUG [Thread-656 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:41:21,694 DEBUG [Thread-656 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:41:21,694 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T10:41:21,697 INFO [Thread-660 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-660. 2024-12-09T10:41:21,697 DEBUG [Thread-660 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T10:41:21,697 DEBUG [Thread-660 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:41:21,697 DEBUG [Thread-660 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:41:21,697 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T10:41:21,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-09T10:41:21,890 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:21,890 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:21,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54262 deadline: 1733740941889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:21,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54216 deadline: 1733740941889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:21,890 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:21,890 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:21,890 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:21,890 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:21,890 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:21,890 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:22,021 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/C/237ac8d1a5a04bc9a600932891c6d48b 2024-12-09T10:41:22,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/A/7ab10fca724f4f6680f3698ad347b254 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/7ab10fca724f4f6680f3698ad347b254 2024-12-09T10:41:22,029 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/7ab10fca724f4f6680f3698ad347b254, entries=150, sequenceid=360, filesize=30.5 K 2024-12-09T10:41:22,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/B/cd5a41d5e4334ae8a558da3feea275d2 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/cd5a41d5e4334ae8a558da3feea275d2 2024-12-09T10:41:22,030 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:22,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54232 deadline: 1733740942029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:22,030 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 7ea43b39fc24,36101,1733740827678 for 'test_row_1' in 0fa6aa84c721c187cb4059599cabb328 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T10:41:22,031 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:22,031 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:22,031 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:41:22,033 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/cd5a41d5e4334ae8a558da3feea275d2, entries=150, sequenceid=360, filesize=12.0 K 2024-12-09T10:41:22,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/C/237ac8d1a5a04bc9a600932891c6d48b as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/237ac8d1a5a04bc9a600932891c6d48b 2024-12-09T10:41:22,037 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/237ac8d1a5a04bc9a600932891c6d48b, entries=150, sequenceid=360, filesize=12.0 K 2024-12-09T10:41:22,038 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 0fa6aa84c721c187cb4059599cabb328 in 964ms, sequenceid=360, compaction requested=false 2024-12-09T10:41:22,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for 0fa6aa84c721c187cb4059599cabb328: 2024-12-09T10:41:22,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. 2024-12-09T10:41:22,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-12-09T10:41:22,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-12-09T10:41:22,041 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-12-09T10:41:22,041 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4270 sec 2024-12-09T10:41:22,042 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 1.4310 sec 2024-12-09T10:41:22,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(8855): Flush requested on 0fa6aa84c721c187cb4059599cabb328 2024-12-09T10:41:22,401 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0fa6aa84c721c187cb4059599cabb328 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-09T10:41:22,402 INFO [Thread-647 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-647. 2024-12-09T10:41:22,402 INFO [Thread-653 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-653. 2024-12-09T10:41:22,402 DEBUG [Thread-647 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T10:41:22,402 DEBUG [Thread-653 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T10:41:22,402 DEBUG [Thread-647 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:41:22,402 DEBUG [Thread-653 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:41:22,402 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fa6aa84c721c187cb4059599cabb328, store=A 2024-12-09T10:41:22,402 DEBUG [Thread-647 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:41:22,402 DEBUG [Thread-653 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:41:22,402 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:41:22,402 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fa6aa84c721c187cb4059599cabb328, store=B 2024-12-09T10:41:22,402 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T10:41:22,402 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:41:22,402 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fa6aa84c721c187cb4059599cabb328, store=C 2024-12-09T10:41:22,402 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:41:22,402 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T10:41:22,408 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209d566a027ff5a4b91b623c3156591ef32_0fa6aa84c721c187cb4059599cabb328 is 50, key is test_row_0/A:col10/1733740882400/Put/seqid=0 2024-12-09T10:41:22,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742050_1226 (size=12454) 2024-12-09T10:41:22,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-09T10:41:22,770 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T10:41:22,812 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:22,816 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209d566a027ff5a4b91b623c3156591ef32_0fa6aa84c721c187cb4059599cabb328 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209d566a027ff5a4b91b623c3156591ef32_0fa6aa84c721c187cb4059599cabb328 2024-12-09T10:41:22,818 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/A/b94ce3ebf0154d55ac164dbe3c563b26, store: [table=TestAcidGuarantees family=A region=0fa6aa84c721c187cb4059599cabb328] 2024-12-09T10:41:22,818 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/A/b94ce3ebf0154d55ac164dbe3c563b26 is 175, key is test_row_0/A:col10/1733740882400/Put/seqid=0 2024-12-09T10:41:22,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742051_1227 (size=31255) 2024-12-09T10:41:23,224 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=381, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/A/b94ce3ebf0154d55ac164dbe3c563b26 2024-12-09T10:41:23,233 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/B/288657bb1400430ba676667e668b41a3 is 50, key is test_row_0/B:col10/1733740882400/Put/seqid=0 2024-12-09T10:41:23,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742052_1228 (size=12301) 2024-12-09T10:41:23,637 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=381 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/B/288657bb1400430ba676667e668b41a3 2024-12-09T10:41:23,653 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/C/e01abc6a8d48495195e8cde94cda1804 is 50, key is test_row_0/C:col10/1733740882400/Put/seqid=0 2024-12-09T10:41:23,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742053_1229 (size=12301) 2024-12-09T10:41:24,058 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=381 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/C/e01abc6a8d48495195e8cde94cda1804 2024-12-09T10:41:24,064 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/A/b94ce3ebf0154d55ac164dbe3c563b26 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/b94ce3ebf0154d55ac164dbe3c563b26 2024-12-09T10:41:24,068 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/b94ce3ebf0154d55ac164dbe3c563b26, entries=150, sequenceid=381, filesize=30.5 K 2024-12-09T10:41:24,069 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/B/288657bb1400430ba676667e668b41a3 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/288657bb1400430ba676667e668b41a3 2024-12-09T10:41:24,073 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/288657bb1400430ba676667e668b41a3, entries=150, sequenceid=381, filesize=12.0 K 2024-12-09T10:41:24,074 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/C/e01abc6a8d48495195e8cde94cda1804 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/e01abc6a8d48495195e8cde94cda1804 2024-12-09T10:41:24,078 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/e01abc6a8d48495195e8cde94cda1804, entries=150, sequenceid=381, filesize=12.0 K 2024-12-09T10:41:24,079 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=0 B/0 for 0fa6aa84c721c187cb4059599cabb328 in 1678ms, sequenceid=381, compaction requested=true 2024-12-09T10:41:24,079 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0fa6aa84c721c187cb4059599cabb328: 2024-12-09T10:41:24,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fa6aa84c721c187cb4059599cabb328:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T10:41:24,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:41:24,079 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T10:41:24,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fa6aa84c721c187cb4059599cabb328:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T10:41:24,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:41:24,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fa6aa84c721c187cb4059599cabb328:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T10:41:24,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T10:41:24,079 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T10:41:24,080 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94515 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T10:41:24,080 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1541): 0fa6aa84c721c187cb4059599cabb328/A is initiating minor compaction (all files) 2024-12-09T10:41:24,080 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T10:41:24,080 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fa6aa84c721c187cb4059599cabb328/A in TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. 2024-12-09T10:41:24,080 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1541): 0fa6aa84c721c187cb4059599cabb328/B is initiating minor compaction (all files) 2024-12-09T10:41:24,080 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/53897573d28b48db984f709c6f269cfd, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/7ab10fca724f4f6680f3698ad347b254, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/b94ce3ebf0154d55ac164dbe3c563b26] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp, totalSize=92.3 K 2024-12-09T10:41:24,080 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. 2024-12-09T10:41:24,080 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fa6aa84c721c187cb4059599cabb328/B in TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. 2024-12-09T10:41:24,080 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. files: [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/53897573d28b48db984f709c6f269cfd, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/7ab10fca724f4f6680f3698ad347b254, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/b94ce3ebf0154d55ac164dbe3c563b26] 2024-12-09T10:41:24,080 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/4de14da0877545ce8ffb2bb45e207b62, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/cd5a41d5e4334ae8a558da3feea275d2, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/288657bb1400430ba676667e668b41a3] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp, totalSize=36.8 K 2024-12-09T10:41:24,081 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 53897573d28b48db984f709c6f269cfd, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1733740878923 2024-12-09T10:41:24,081 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 4de14da0877545ce8ffb2bb45e207b62, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1733740878923 2024-12-09T10:41:24,081 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7ab10fca724f4f6680f3698ad347b254, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1733740880086 2024-12-09T10:41:24,081 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting cd5a41d5e4334ae8a558da3feea275d2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1733740880086 2024-12-09T10:41:24,081 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting b94ce3ebf0154d55ac164dbe3c563b26, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1733740881251 2024-12-09T10:41:24,081 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 288657bb1400430ba676667e668b41a3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1733740881251 2024-12-09T10:41:24,094 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fa6aa84c721c187cb4059599cabb328#B#compaction#189 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:41:24,095 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/B/64c415bd024440d39562c89f6ceb721f is 50, key is test_row_0/B:col10/1733740882400/Put/seqid=0 2024-12-09T10:41:24,107 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0fa6aa84c721c187cb4059599cabb328] 2024-12-09T10:41:24,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742054_1230 (size=13153) 2024-12-09T10:41:24,117 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/B/64c415bd024440d39562c89f6ceb721f as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/64c415bd024440d39562c89f6ceb721f 2024-12-09T10:41:24,124 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0fa6aa84c721c187cb4059599cabb328/B of 0fa6aa84c721c187cb4059599cabb328 into 64c415bd024440d39562c89f6ceb721f(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:41:24,124 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fa6aa84c721c187cb4059599cabb328: 2024-12-09T10:41:24,124 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., storeName=0fa6aa84c721c187cb4059599cabb328/B, priority=13, startTime=1733740884079; duration=0sec 2024-12-09T10:41:24,124 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T10:41:24,124 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fa6aa84c721c187cb4059599cabb328:B 2024-12-09T10:41:24,125 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120971e7ca9af9cf465d8e160441ab0eb2f8_0fa6aa84c721c187cb4059599cabb328 store=[table=TestAcidGuarantees family=A region=0fa6aa84c721c187cb4059599cabb328] 2024-12-09T10:41:24,128 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T10:41:24,129 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T10:41:24,130 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1541): 0fa6aa84c721c187cb4059599cabb328/C is initiating minor compaction (all files) 2024-12-09T10:41:24,130 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fa6aa84c721c187cb4059599cabb328/C in TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. 2024-12-09T10:41:24,130 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/7111523abe794d46b92959727e070e09, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/237ac8d1a5a04bc9a600932891c6d48b, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/e01abc6a8d48495195e8cde94cda1804] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp, totalSize=36.8 K 2024-12-09T10:41:24,130 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 7111523abe794d46b92959727e070e09, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1733740878923 2024-12-09T10:41:24,131 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 237ac8d1a5a04bc9a600932891c6d48b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1733740880086 2024-12-09T10:41:24,131 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting e01abc6a8d48495195e8cde94cda1804, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1733740881251 2024-12-09T10:41:24,156 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fa6aa84c721c187cb4059599cabb328#C#compaction#191 average throughput is 0.66 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:41:24,157 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/C/b5c5c863408f48cc9b49c9c694ab2d8e is 50, key is test_row_0/C:col10/1733740882400/Put/seqid=0 2024-12-09T10:41:24,171 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120971e7ca9af9cf465d8e160441ab0eb2f8_0fa6aa84c721c187cb4059599cabb328, store=[table=TestAcidGuarantees family=A region=0fa6aa84c721c187cb4059599cabb328] 2024-12-09T10:41:24,171 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120971e7ca9af9cf465d8e160441ab0eb2f8_0fa6aa84c721c187cb4059599cabb328 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0fa6aa84c721c187cb4059599cabb328] 2024-12-09T10:41:24,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742055_1231 (size=13153) 2024-12-09T10:41:24,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742056_1232 (size=4469) 2024-12-09T10:41:24,207 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/C/b5c5c863408f48cc9b49c9c694ab2d8e as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/b5c5c863408f48cc9b49c9c694ab2d8e 2024-12-09T10:41:24,212 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0fa6aa84c721c187cb4059599cabb328/C of 0fa6aa84c721c187cb4059599cabb328 into b5c5c863408f48cc9b49c9c694ab2d8e(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:41:24,212 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fa6aa84c721c187cb4059599cabb328: 2024-12-09T10:41:24,212 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., storeName=0fa6aa84c721c187cb4059599cabb328/C, priority=13, startTime=1733740884079; duration=0sec 2024-12-09T10:41:24,212 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:41:24,212 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fa6aa84c721c187cb4059599cabb328:C 2024-12-09T10:41:24,602 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fa6aa84c721c187cb4059599cabb328#A#compaction#190 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:41:24,603 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/A/26bf77012fd64fe38dae95835e7cafe3 is 175, key is test_row_0/A:col10/1733740882400/Put/seqid=0 2024-12-09T10:41:24,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742057_1233 (size=32107) 2024-12-09T10:41:24,612 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/A/26bf77012fd64fe38dae95835e7cafe3 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/26bf77012fd64fe38dae95835e7cafe3 2024-12-09T10:41:24,617 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0fa6aa84c721c187cb4059599cabb328/A of 0fa6aa84c721c187cb4059599cabb328 into 26bf77012fd64fe38dae95835e7cafe3(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:41:24,617 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fa6aa84c721c187cb4059599cabb328: 2024-12-09T10:41:24,617 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., storeName=0fa6aa84c721c187cb4059599cabb328/A, priority=13, startTime=1733740884079; duration=0sec 2024-12-09T10:41:24,617 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:41:24,617 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fa6aa84c721c187cb4059599cabb328:A 2024-12-09T10:41:25,933 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-09T10:41:30,231 INFO [Thread-649 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-649. 2024-12-09T10:41:30,231 DEBUG [Thread-649 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T10:41:30,231 DEBUG [Thread-649 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:41:30,231 DEBUG [Thread-649 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:41:30,231 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T10:41:30,266 INFO [Thread-645 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-645. 2024-12-09T10:41:30,267 DEBUG [Thread-645 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T10:41:30,267 DEBUG [Thread-645 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:41:30,267 DEBUG [Thread-645 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:41:30,267 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T10:41:32,101 INFO [Thread-651 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-651. 2024-12-09T10:41:32,101 DEBUG [Thread-651 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T10:41:32,101 DEBUG [Thread-651 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:41:32,102 DEBUG [Thread-651 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:41:32,102 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T10:41:32,102 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-09T10:41:32,102 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 8 2024-12-09T10:41:32,102 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 115 2024-12-09T10:41:32,102 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 8 2024-12-09T10:41:32,102 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 73 2024-12-09T10:41:32,102 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 101 2024-12-09T10:41:32,102 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-09T10:41:32,102 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2789 2024-12-09T10:41:32,102 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2908 2024-12-09T10:41:32,102 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-09T10:41:32,102 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1245 2024-12-09T10:41:32,102 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3735 rows 2024-12-09T10:41:32,102 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1260 2024-12-09T10:41:32,102 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3769 rows 2024-12-09T10:41:32,102 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-09T10:41:32,102 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T10:41:32,102 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:41:32,102 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:41:32,102 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T10:41:32,103 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-12-09T10:41:32,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-09T10:41:32,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-09T10:41:32,106 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733740892106"}]},"ts":"1733740892106"} 2024-12-09T10:41:32,107 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-09T10:41:32,107 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-09T10:41:32,108 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-09T10:41:32,109 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=54, ppid=53, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0fa6aa84c721c187cb4059599cabb328, UNASSIGN}] 2024-12-09T10:41:32,110 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=54, ppid=53, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0fa6aa84c721c187cb4059599cabb328, UNASSIGN 2024-12-09T10:41:32,110 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=54 updating hbase:meta row=0fa6aa84c721c187cb4059599cabb328, regionState=CLOSING, regionLocation=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:32,111 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=54, ppid=53, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0fa6aa84c721c187cb4059599cabb328, UNASSIGN because future has completed 2024-12-09T10:41:32,112 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-09T10:41:32,112 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678}] 2024-12-09T10:41:32,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-09T10:41:32,264 INFO [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] handler.UnassignRegionHandler(122): Close 0fa6aa84c721c187cb4059599cabb328 2024-12-09T10:41:32,264 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-09T10:41:32,264 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1722): Closing 0fa6aa84c721c187cb4059599cabb328, disabling compactions & flushes 2024-12-09T10:41:32,264 INFO [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. 2024-12-09T10:41:32,264 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. 2024-12-09T10:41:32,264 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. after waiting 0 ms 2024-12-09T10:41:32,264 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. 2024-12-09T10:41:32,264 INFO [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(2902): Flushing 0fa6aa84c721c187cb4059599cabb328 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-12-09T10:41:32,265 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fa6aa84c721c187cb4059599cabb328, store=A 2024-12-09T10:41:32,265 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:41:32,265 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fa6aa84c721c187cb4059599cabb328, store=B 2024-12-09T10:41:32,265 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:41:32,265 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fa6aa84c721c187cb4059599cabb328, store=C 2024-12-09T10:41:32,265 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:41:32,271 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209c552771bbcbf4b2192fcb1a4f948976b_0fa6aa84c721c187cb4059599cabb328 is 50, key is test_row_1/A:col10/1733740892100/Put/seqid=0 2024-12-09T10:41:32,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742058_1234 (size=9914) 2024-12-09T10:41:32,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-09T10:41:32,675 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:32,680 INFO [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209c552771bbcbf4b2192fcb1a4f948976b_0fa6aa84c721c187cb4059599cabb328 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209c552771bbcbf4b2192fcb1a4f948976b_0fa6aa84c721c187cb4059599cabb328 2024-12-09T10:41:32,681 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/A/790ea86e14d5451a9cb6fa841d9758a5, store: [table=TestAcidGuarantees family=A region=0fa6aa84c721c187cb4059599cabb328] 2024-12-09T10:41:32,681 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/A/790ea86e14d5451a9cb6fa841d9758a5 is 175, key is test_row_1/A:col10/1733740892100/Put/seqid=0 2024-12-09T10:41:32,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742059_1235 (size=22561) 2024-12-09T10:41:32,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-09T10:41:33,085 INFO [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=390, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/A/790ea86e14d5451a9cb6fa841d9758a5 2024-12-09T10:41:33,092 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/B/40343afbeb2e40b79357074e8ece34db is 50, key is test_row_1/B:col10/1733740892100/Put/seqid=0 2024-12-09T10:41:33,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742060_1236 (size=9857) 2024-12-09T10:41:33,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-09T10:41:33,496 INFO [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=390 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/B/40343afbeb2e40b79357074e8ece34db 2024-12-09T10:41:33,503 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/C/ce902411be0b410fa01ea344364123da is 50, key is test_row_1/C:col10/1733740892100/Put/seqid=0 2024-12-09T10:41:33,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742061_1237 (size=9857) 2024-12-09T10:41:33,907 INFO [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=390 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/C/ce902411be0b410fa01ea344364123da 2024-12-09T10:41:33,912 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/A/790ea86e14d5451a9cb6fa841d9758a5 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/790ea86e14d5451a9cb6fa841d9758a5 2024-12-09T10:41:33,916 INFO [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/790ea86e14d5451a9cb6fa841d9758a5, entries=100, sequenceid=390, filesize=22.0 K 2024-12-09T10:41:33,916 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/B/40343afbeb2e40b79357074e8ece34db as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/40343afbeb2e40b79357074e8ece34db 2024-12-09T10:41:33,920 INFO [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/40343afbeb2e40b79357074e8ece34db, entries=100, sequenceid=390, filesize=9.6 K 2024-12-09T10:41:33,921 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/.tmp/C/ce902411be0b410fa01ea344364123da as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/ce902411be0b410fa01ea344364123da 2024-12-09T10:41:33,924 INFO [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/ce902411be0b410fa01ea344364123da, entries=100, sequenceid=390, filesize=9.6 K 2024-12-09T10:41:33,925 INFO [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 0fa6aa84c721c187cb4059599cabb328 in 1661ms, sequenceid=390, compaction requested=false 2024-12-09T10:41:33,925 DEBUG [StoreCloser-TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/bc723ed71c7746e6ba416877d93deb11, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/065f49e028744d5f809432dff47e16b8, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/83b62a9b2fff451c97a86f0d872e95b8, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/020404d7408f4ec8a127f790ef84bce0, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/42ff5e8e429f425fbc3dbfdb0886d395, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/2b6b3c49586f44ebb4f32a59d05de2a2, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/3501a6d4b92846adbf898a541db560e1, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/48da3f0ab8d14e0292ba1d69cc18a75f, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/a1d197bf4e8d42759b05f637fc6637d9, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/d37dada0d4f5479cb23d328ef5d6a3c1, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/82ce835e9a94447ca78a65844390dd50, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/6821b37b3d82407b9fdcc9e5aa501492, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/3a4dbebfc45f454e9bbcfcd797a67898, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/58c1596b9ee442b99d82568c92184ea2, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/40f11f3552b542039dc14066a37d1a92, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/6b1c48de54e640aeaec09f3d920d2801, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/49529cf6fe984a33b2f0e31b6b07806f, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/f807342bd7df4170b93ac61f55053f2d, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/d0bfa67bfa2747309f337fef8c9fa553, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/b4256eab5ebc4ca2ae7b1900a961b509, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/ace48c0b600a4403af32be9a228cc732, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/b4e47bcb3a544cb1ab0c67aec1ee7df0, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/53897573d28b48db984f709c6f269cfd, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/7ab10fca724f4f6680f3698ad347b254, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/b94ce3ebf0154d55ac164dbe3c563b26] to archive 2024-12-09T10:41:33,926 DEBUG [StoreCloser-TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T10:41:33,928 DEBUG [StoreCloser-TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/bc723ed71c7746e6ba416877d93deb11 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/bc723ed71c7746e6ba416877d93deb11 2024-12-09T10:41:33,929 DEBUG [StoreCloser-TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/065f49e028744d5f809432dff47e16b8 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/065f49e028744d5f809432dff47e16b8 2024-12-09T10:41:33,930 DEBUG [StoreCloser-TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/83b62a9b2fff451c97a86f0d872e95b8 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/83b62a9b2fff451c97a86f0d872e95b8 2024-12-09T10:41:33,931 DEBUG [StoreCloser-TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/020404d7408f4ec8a127f790ef84bce0 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/020404d7408f4ec8a127f790ef84bce0 2024-12-09T10:41:33,932 DEBUG [StoreCloser-TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/42ff5e8e429f425fbc3dbfdb0886d395 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/42ff5e8e429f425fbc3dbfdb0886d395 2024-12-09T10:41:33,932 DEBUG [StoreCloser-TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/2b6b3c49586f44ebb4f32a59d05de2a2 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/2b6b3c49586f44ebb4f32a59d05de2a2 2024-12-09T10:41:33,933 DEBUG [StoreCloser-TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/3501a6d4b92846adbf898a541db560e1 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/3501a6d4b92846adbf898a541db560e1 2024-12-09T10:41:33,934 DEBUG [StoreCloser-TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/48da3f0ab8d14e0292ba1d69cc18a75f to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/48da3f0ab8d14e0292ba1d69cc18a75f 2024-12-09T10:41:33,935 DEBUG [StoreCloser-TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/a1d197bf4e8d42759b05f637fc6637d9 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/a1d197bf4e8d42759b05f637fc6637d9 2024-12-09T10:41:33,936 DEBUG [StoreCloser-TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/d37dada0d4f5479cb23d328ef5d6a3c1 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/d37dada0d4f5479cb23d328ef5d6a3c1 2024-12-09T10:41:33,937 DEBUG [StoreCloser-TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/82ce835e9a94447ca78a65844390dd50 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/82ce835e9a94447ca78a65844390dd50 2024-12-09T10:41:33,938 DEBUG [StoreCloser-TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/6821b37b3d82407b9fdcc9e5aa501492 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/6821b37b3d82407b9fdcc9e5aa501492 2024-12-09T10:41:33,939 DEBUG [StoreCloser-TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/3a4dbebfc45f454e9bbcfcd797a67898 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/3a4dbebfc45f454e9bbcfcd797a67898 2024-12-09T10:41:33,940 DEBUG [StoreCloser-TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/58c1596b9ee442b99d82568c92184ea2 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/58c1596b9ee442b99d82568c92184ea2 2024-12-09T10:41:33,941 DEBUG [StoreCloser-TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/40f11f3552b542039dc14066a37d1a92 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/40f11f3552b542039dc14066a37d1a92 2024-12-09T10:41:33,942 DEBUG [StoreCloser-TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/6b1c48de54e640aeaec09f3d920d2801 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/6b1c48de54e640aeaec09f3d920d2801 2024-12-09T10:41:33,943 DEBUG [StoreCloser-TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/49529cf6fe984a33b2f0e31b6b07806f to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/49529cf6fe984a33b2f0e31b6b07806f 2024-12-09T10:41:33,943 DEBUG [StoreCloser-TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/f807342bd7df4170b93ac61f55053f2d to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/f807342bd7df4170b93ac61f55053f2d 2024-12-09T10:41:33,944 DEBUG [StoreCloser-TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/d0bfa67bfa2747309f337fef8c9fa553 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/d0bfa67bfa2747309f337fef8c9fa553 2024-12-09T10:41:33,945 DEBUG [StoreCloser-TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/b4256eab5ebc4ca2ae7b1900a961b509 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/b4256eab5ebc4ca2ae7b1900a961b509 2024-12-09T10:41:33,946 DEBUG [StoreCloser-TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/ace48c0b600a4403af32be9a228cc732 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/ace48c0b600a4403af32be9a228cc732 2024-12-09T10:41:33,947 DEBUG [StoreCloser-TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/b4e47bcb3a544cb1ab0c67aec1ee7df0 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/b4e47bcb3a544cb1ab0c67aec1ee7df0 2024-12-09T10:41:33,948 DEBUG [StoreCloser-TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/53897573d28b48db984f709c6f269cfd to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/53897573d28b48db984f709c6f269cfd 2024-12-09T10:41:33,949 DEBUG [StoreCloser-TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/7ab10fca724f4f6680f3698ad347b254 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/7ab10fca724f4f6680f3698ad347b254 2024-12-09T10:41:33,950 DEBUG [StoreCloser-TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/b94ce3ebf0154d55ac164dbe3c563b26 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/b94ce3ebf0154d55ac164dbe3c563b26 2024-12-09T10:41:33,952 DEBUG [StoreCloser-TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/87470e4e2fba4d09870884bc4640d990, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/a2496d8710d44be7bce45d45f33129ac, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/3f4806145fc54f58b9f0a4583c0cff33, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/e47aae86e999456c9612f2584ca500e8, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/1842dc69218641258ea2234ff46b6062, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/6069a9d574d84c66a94f2b6f9f3797e5, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/38f7012ca3c149a7a6beb6a87706e066, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/eff84c229d6549f4b952f5df1d3bbd9b, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/18c3d288358244e1b5726b8bfb2d358a, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/d78df51593eb49c3944729d10ea66e1b, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/ce268d9901714d71943d9cd673573ce2, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/0d9b62035cb04cbfb351669260902a8e, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/19da3decfe974d69a605d61478e5dfa2, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/b72f801956bd46fc98b5a8a195194465, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/c4ecdc6ece1f4265902e0ff3c4bbbd61, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/1e0a6269cead473692137011866cd376, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/ef18e1bcd1a5437890c7958856a67cd2, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/8fee427cf86b4c70b796ba918f2b30c0, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/0ec8ff9bb1fc4d05b34777572b2e677f, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/efa1279dd03d4046a1437c417efc442d, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/6087f5b8160d49d9aba0aa435d257b4f, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/4de14da0877545ce8ffb2bb45e207b62, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/445420969aa342859444af5c63e864c2, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/cd5a41d5e4334ae8a558da3feea275d2, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/288657bb1400430ba676667e668b41a3] to archive 2024-12-09T10:41:33,952 DEBUG [StoreCloser-TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T10:41:33,954 DEBUG [StoreCloser-TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/87470e4e2fba4d09870884bc4640d990 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/87470e4e2fba4d09870884bc4640d990 2024-12-09T10:41:33,955 DEBUG [StoreCloser-TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/a2496d8710d44be7bce45d45f33129ac to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/a2496d8710d44be7bce45d45f33129ac 2024-12-09T10:41:33,956 DEBUG [StoreCloser-TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/3f4806145fc54f58b9f0a4583c0cff33 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/3f4806145fc54f58b9f0a4583c0cff33 2024-12-09T10:41:33,957 DEBUG [StoreCloser-TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/e47aae86e999456c9612f2584ca500e8 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/e47aae86e999456c9612f2584ca500e8 2024-12-09T10:41:33,958 DEBUG [StoreCloser-TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/1842dc69218641258ea2234ff46b6062 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/1842dc69218641258ea2234ff46b6062 2024-12-09T10:41:33,959 DEBUG [StoreCloser-TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/6069a9d574d84c66a94f2b6f9f3797e5 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/6069a9d574d84c66a94f2b6f9f3797e5 2024-12-09T10:41:33,960 DEBUG [StoreCloser-TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/38f7012ca3c149a7a6beb6a87706e066 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/38f7012ca3c149a7a6beb6a87706e066 2024-12-09T10:41:33,961 DEBUG [StoreCloser-TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/eff84c229d6549f4b952f5df1d3bbd9b to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/eff84c229d6549f4b952f5df1d3bbd9b 2024-12-09T10:41:33,962 DEBUG [StoreCloser-TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/18c3d288358244e1b5726b8bfb2d358a to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/18c3d288358244e1b5726b8bfb2d358a 2024-12-09T10:41:33,963 DEBUG [StoreCloser-TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/d78df51593eb49c3944729d10ea66e1b to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/d78df51593eb49c3944729d10ea66e1b 2024-12-09T10:41:33,964 DEBUG [StoreCloser-TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/ce268d9901714d71943d9cd673573ce2 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/ce268d9901714d71943d9cd673573ce2 2024-12-09T10:41:33,964 DEBUG [StoreCloser-TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/0d9b62035cb04cbfb351669260902a8e to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/0d9b62035cb04cbfb351669260902a8e 2024-12-09T10:41:33,965 DEBUG [StoreCloser-TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/19da3decfe974d69a605d61478e5dfa2 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/19da3decfe974d69a605d61478e5dfa2 2024-12-09T10:41:33,966 DEBUG [StoreCloser-TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/b72f801956bd46fc98b5a8a195194465 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/b72f801956bd46fc98b5a8a195194465 2024-12-09T10:41:33,967 DEBUG [StoreCloser-TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/c4ecdc6ece1f4265902e0ff3c4bbbd61 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/c4ecdc6ece1f4265902e0ff3c4bbbd61 2024-12-09T10:41:33,968 DEBUG [StoreCloser-TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/1e0a6269cead473692137011866cd376 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/1e0a6269cead473692137011866cd376 2024-12-09T10:41:33,969 DEBUG [StoreCloser-TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/ef18e1bcd1a5437890c7958856a67cd2 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/ef18e1bcd1a5437890c7958856a67cd2 2024-12-09T10:41:33,970 DEBUG [StoreCloser-TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/8fee427cf86b4c70b796ba918f2b30c0 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/8fee427cf86b4c70b796ba918f2b30c0 2024-12-09T10:41:33,971 DEBUG [StoreCloser-TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/0ec8ff9bb1fc4d05b34777572b2e677f to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/0ec8ff9bb1fc4d05b34777572b2e677f 2024-12-09T10:41:33,972 DEBUG [StoreCloser-TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/efa1279dd03d4046a1437c417efc442d to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/efa1279dd03d4046a1437c417efc442d 2024-12-09T10:41:33,973 DEBUG [StoreCloser-TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/6087f5b8160d49d9aba0aa435d257b4f to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/6087f5b8160d49d9aba0aa435d257b4f 2024-12-09T10:41:33,974 DEBUG [StoreCloser-TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/4de14da0877545ce8ffb2bb45e207b62 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/4de14da0877545ce8ffb2bb45e207b62 2024-12-09T10:41:33,975 DEBUG [StoreCloser-TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/445420969aa342859444af5c63e864c2 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/445420969aa342859444af5c63e864c2 2024-12-09T10:41:33,976 DEBUG [StoreCloser-TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/cd5a41d5e4334ae8a558da3feea275d2 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/cd5a41d5e4334ae8a558da3feea275d2 2024-12-09T10:41:33,977 DEBUG [StoreCloser-TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/288657bb1400430ba676667e668b41a3 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/288657bb1400430ba676667e668b41a3 2024-12-09T10:41:33,978 DEBUG [StoreCloser-TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/b0ecbb1814bf493aaa4d0451511c7a27, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/f6a3d5a1078442b3aaca48d8dae5c589, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/8c584be347c84b2f89f7c5ffe64d8606, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/8efe3a6694894bb685fc80b4009ce556, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/3039218d1e594eb0901f4c6de283a328, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/1440e526f8594bc5b88e2452608ba41a, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/6461152a96f94a269f9beff2cd9c976d, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/a926c57ec3df468b9ff28e3ae92be234, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/96e005858a5d45d2a2e13bcf46613134, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/9d86a1a56fec43039e9f9eaea02fb98f, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/6f36b59f111e421087329cd42721f416, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/0ae236d8169f4ff2ac3b9945a24449e7, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/03eeddf98a654f10b0f01e80caf82eb5, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/cfd836c1bff34bfca0311c83b92b5018, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/553ba76ed01f4450b7c7bcbfc466dcaf, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/2ec0aea4ab3a4962ba33fd9ff364de21, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/c56fd681b92a4616bdd927876145c32a, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/47cf06ed32ee4f35acd7479ab214efe2, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/7d16e799c47b43dc91349e9cc62324a3, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/5a68f4b86db14504a06085f9546fba96, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/b98a7d52903c416a80a520be89f14a08, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/7111523abe794d46b92959727e070e09, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/dd9629d51cc44451bca486924e80a08a, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/237ac8d1a5a04bc9a600932891c6d48b, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/e01abc6a8d48495195e8cde94cda1804] to archive 2024-12-09T10:41:33,979 DEBUG [StoreCloser-TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T10:41:33,980 DEBUG [StoreCloser-TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/b0ecbb1814bf493aaa4d0451511c7a27 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/b0ecbb1814bf493aaa4d0451511c7a27 2024-12-09T10:41:33,981 DEBUG [StoreCloser-TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/f6a3d5a1078442b3aaca48d8dae5c589 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/f6a3d5a1078442b3aaca48d8dae5c589 2024-12-09T10:41:33,982 DEBUG [StoreCloser-TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/8c584be347c84b2f89f7c5ffe64d8606 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/8c584be347c84b2f89f7c5ffe64d8606 2024-12-09T10:41:33,983 DEBUG [StoreCloser-TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/8efe3a6694894bb685fc80b4009ce556 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/8efe3a6694894bb685fc80b4009ce556 2024-12-09T10:41:33,984 DEBUG [StoreCloser-TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/3039218d1e594eb0901f4c6de283a328 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/3039218d1e594eb0901f4c6de283a328 2024-12-09T10:41:33,985 DEBUG [StoreCloser-TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/1440e526f8594bc5b88e2452608ba41a to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/1440e526f8594bc5b88e2452608ba41a 2024-12-09T10:41:33,986 DEBUG [StoreCloser-TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/6461152a96f94a269f9beff2cd9c976d to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/6461152a96f94a269f9beff2cd9c976d 2024-12-09T10:41:33,987 DEBUG [StoreCloser-TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/a926c57ec3df468b9ff28e3ae92be234 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/a926c57ec3df468b9ff28e3ae92be234 2024-12-09T10:41:33,988 DEBUG [StoreCloser-TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/96e005858a5d45d2a2e13bcf46613134 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/96e005858a5d45d2a2e13bcf46613134 2024-12-09T10:41:33,989 DEBUG [StoreCloser-TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/9d86a1a56fec43039e9f9eaea02fb98f to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/9d86a1a56fec43039e9f9eaea02fb98f 2024-12-09T10:41:33,990 DEBUG [StoreCloser-TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/6f36b59f111e421087329cd42721f416 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/6f36b59f111e421087329cd42721f416 2024-12-09T10:41:33,991 DEBUG [StoreCloser-TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/0ae236d8169f4ff2ac3b9945a24449e7 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/0ae236d8169f4ff2ac3b9945a24449e7 2024-12-09T10:41:33,992 DEBUG [StoreCloser-TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/03eeddf98a654f10b0f01e80caf82eb5 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/03eeddf98a654f10b0f01e80caf82eb5 2024-12-09T10:41:33,993 DEBUG [StoreCloser-TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/cfd836c1bff34bfca0311c83b92b5018 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/cfd836c1bff34bfca0311c83b92b5018 2024-12-09T10:41:33,994 DEBUG [StoreCloser-TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/553ba76ed01f4450b7c7bcbfc466dcaf to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/553ba76ed01f4450b7c7bcbfc466dcaf 2024-12-09T10:41:33,995 DEBUG [StoreCloser-TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/2ec0aea4ab3a4962ba33fd9ff364de21 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/2ec0aea4ab3a4962ba33fd9ff364de21 2024-12-09T10:41:33,996 DEBUG [StoreCloser-TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/c56fd681b92a4616bdd927876145c32a to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/c56fd681b92a4616bdd927876145c32a 2024-12-09T10:41:33,997 DEBUG [StoreCloser-TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/47cf06ed32ee4f35acd7479ab214efe2 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/47cf06ed32ee4f35acd7479ab214efe2 2024-12-09T10:41:33,998 DEBUG [StoreCloser-TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/7d16e799c47b43dc91349e9cc62324a3 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/7d16e799c47b43dc91349e9cc62324a3 2024-12-09T10:41:33,998 DEBUG [StoreCloser-TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/5a68f4b86db14504a06085f9546fba96 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/5a68f4b86db14504a06085f9546fba96 2024-12-09T10:41:33,999 DEBUG [StoreCloser-TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/b98a7d52903c416a80a520be89f14a08 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/b98a7d52903c416a80a520be89f14a08 2024-12-09T10:41:34,001 DEBUG [StoreCloser-TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/7111523abe794d46b92959727e070e09 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/7111523abe794d46b92959727e070e09 2024-12-09T10:41:34,002 DEBUG [StoreCloser-TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/dd9629d51cc44451bca486924e80a08a to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/dd9629d51cc44451bca486924e80a08a 2024-12-09T10:41:34,003 DEBUG [StoreCloser-TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/237ac8d1a5a04bc9a600932891c6d48b to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/237ac8d1a5a04bc9a600932891c6d48b 2024-12-09T10:41:34,004 DEBUG [StoreCloser-TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/e01abc6a8d48495195e8cde94cda1804 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/e01abc6a8d48495195e8cde94cda1804 2024-12-09T10:41:34,008 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/recovered.edits/393.seqid, newMaxSeqId=393, maxSeqId=4 2024-12-09T10:41:34,009 INFO [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. 2024-12-09T10:41:34,009 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1676): Region close journal for 0fa6aa84c721c187cb4059599cabb328: Waiting for close lock at 1733740892264Running coprocessor pre-close hooks at 1733740892264Disabling compacts and flushes for region at 1733740892264Disabling writes for close at 1733740892264Obtaining lock to block concurrent updates at 1733740892264Preparing flush snapshotting stores in 0fa6aa84c721c187cb4059599cabb328 at 1733740892264Finished memstore snapshotting TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1733740892265 (+1 ms)Flushing stores of TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328. at 1733740892265Flushing 0fa6aa84c721c187cb4059599cabb328/A: creating writer at 1733740892265Flushing mob file 0fa6aa84c721c187cb4059599cabb328/A: appending metadata at 1733740892271 (+6 ms)Flushing mob file 0fa6aa84c721c187cb4059599cabb328/A: closing flushed file at 1733740892271Flushing 0fa6aa84c721c187cb4059599cabb328/A: appending metadata at 1733740892681 (+410 ms)Flushing 0fa6aa84c721c187cb4059599cabb328/A: closing flushed file at 1733740892681Flushing 0fa6aa84c721c187cb4059599cabb328/B: creating writer at 1733740893089 (+408 ms)Flushing 0fa6aa84c721c187cb4059599cabb328/B: appending metadata at 1733740893092 (+3 ms)Flushing 0fa6aa84c721c187cb4059599cabb328/B: closing flushed file at 1733740893092Flushing 0fa6aa84c721c187cb4059599cabb328/C: creating writer at 1733740893500 (+408 ms)Flushing 0fa6aa84c721c187cb4059599cabb328/C: appending metadata at 1733740893503 (+3 ms)Flushing 0fa6aa84c721c187cb4059599cabb328/C: closing flushed file at 1733740893503Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@25436056: reopening flushed file at 1733740893911 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@143c8596: reopening flushed file at 1733740893916 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@19cdd380: reopening flushed file at 1733740893920 (+4 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 0fa6aa84c721c187cb4059599cabb328 in 1661ms, sequenceid=390, compaction requested=false at 1733740893925 (+5 ms)Writing region close event to WAL at 1733740894005 (+80 ms)Running coprocessor post-close hooks at 1733740894008 (+3 ms)Closed at 1733740894009 (+1 ms) 2024-12-09T10:41:34,011 INFO [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] handler.UnassignRegionHandler(157): Closed 0fa6aa84c721c187cb4059599cabb328 2024-12-09T10:41:34,011 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=54 updating hbase:meta row=0fa6aa84c721c187cb4059599cabb328, regionState=CLOSED 2024-12-09T10:41:34,013 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=55, ppid=54, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 because future has completed 2024-12-09T10:41:34,015 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-12-09T10:41:34,015 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; CloseRegionProcedure 0fa6aa84c721c187cb4059599cabb328, server=7ea43b39fc24,36101,1733740827678 in 1.9010 sec 2024-12-09T10:41:34,016 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=54, resume processing ppid=53 2024-12-09T10:41:34,016 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, ppid=53, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0fa6aa84c721c187cb4059599cabb328, UNASSIGN in 1.9060 sec 2024-12-09T10:41:34,018 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-12-09T10:41:34,018 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9090 sec 2024-12-09T10:41:34,019 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733740894019"}]},"ts":"1733740894019"} 2024-12-09T10:41:34,020 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-09T10:41:34,020 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-09T10:41:34,022 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9180 sec 2024-12-09T10:41:34,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-09T10:41:34,250 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-09T10:41:34,250 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-12-09T10:41:34,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T10:41:34,252 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=56, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T10:41:34,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-09T10:41:34,253 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=56, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T10:41:34,255 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328 2024-12-09T10:41:34,257 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A, FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B, FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C, FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/recovered.edits] 2024-12-09T10:41:34,260 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/26bf77012fd64fe38dae95835e7cafe3 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/26bf77012fd64fe38dae95835e7cafe3 2024-12-09T10:41:34,261 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/790ea86e14d5451a9cb6fa841d9758a5 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/A/790ea86e14d5451a9cb6fa841d9758a5 2024-12-09T10:41:34,263 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/40343afbeb2e40b79357074e8ece34db to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/40343afbeb2e40b79357074e8ece34db 2024-12-09T10:41:34,264 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/64c415bd024440d39562c89f6ceb721f to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/B/64c415bd024440d39562c89f6ceb721f 2024-12-09T10:41:34,266 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/b5c5c863408f48cc9b49c9c694ab2d8e to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/b5c5c863408f48cc9b49c9c694ab2d8e 2024-12-09T10:41:34,267 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/ce902411be0b410fa01ea344364123da to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/C/ce902411be0b410fa01ea344364123da 2024-12-09T10:41:34,270 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/recovered.edits/393.seqid to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328/recovered.edits/393.seqid 2024-12-09T10:41:34,271 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/0fa6aa84c721c187cb4059599cabb328 2024-12-09T10:41:34,271 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-09T10:41:34,271 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-09T10:41:34,272 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-09T10:41:34,275 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412090997665fed9c445b8e1ba18f6b9922d8_0fa6aa84c721c187cb4059599cabb328 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412090997665fed9c445b8e1ba18f6b9922d8_0fa6aa84c721c187cb4059599cabb328 2024-12-09T10:41:34,276 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209116a59b43b34490e9af663e8715ba3a3_0fa6aa84c721c187cb4059599cabb328 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209116a59b43b34490e9af663e8715ba3a3_0fa6aa84c721c187cb4059599cabb328 2024-12-09T10:41:34,277 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120925d84e6264844f07a0936306b40f8f91_0fa6aa84c721c187cb4059599cabb328 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120925d84e6264844f07a0936306b40f8f91_0fa6aa84c721c187cb4059599cabb328 2024-12-09T10:41:34,278 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412092d4f5d6929404a698659ea9fef0b066d_0fa6aa84c721c187cb4059599cabb328 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412092d4f5d6929404a698659ea9fef0b066d_0fa6aa84c721c187cb4059599cabb328 2024-12-09T10:41:34,279 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412092d8f59ded98b47b4a02b655b0abee415_0fa6aa84c721c187cb4059599cabb328 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412092d8f59ded98b47b4a02b655b0abee415_0fa6aa84c721c187cb4059599cabb328 2024-12-09T10:41:34,280 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412093e840cec029f4335a7430a4d057be534_0fa6aa84c721c187cb4059599cabb328 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412093e840cec029f4335a7430a4d057be534_0fa6aa84c721c187cb4059599cabb328 2024-12-09T10:41:34,281 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120966a0e3cea85443878a1b0e56a0978004_0fa6aa84c721c187cb4059599cabb328 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120966a0e3cea85443878a1b0e56a0978004_0fa6aa84c721c187cb4059599cabb328 2024-12-09T10:41:34,282 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120979be322abbd04fb3b4773e5d7215d0ea_0fa6aa84c721c187cb4059599cabb328 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120979be322abbd04fb3b4773e5d7215d0ea_0fa6aa84c721c187cb4059599cabb328 2024-12-09T10:41:34,283 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120986f6747554744423b83c3157ed10afa4_0fa6aa84c721c187cb4059599cabb328 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120986f6747554744423b83c3157ed10afa4_0fa6aa84c721c187cb4059599cabb328 2024-12-09T10:41:34,285 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209936f247f81ce488ab9b5d80f14186783_0fa6aa84c721c187cb4059599cabb328 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209936f247f81ce488ab9b5d80f14186783_0fa6aa84c721c187cb4059599cabb328 2024-12-09T10:41:34,286 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120998376236ff40403f94538b4a04975c79_0fa6aa84c721c187cb4059599cabb328 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120998376236ff40403f94538b4a04975c79_0fa6aa84c721c187cb4059599cabb328 2024-12-09T10:41:34,287 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412099bd4979dd02b4d269cf58e8a365f2011_0fa6aa84c721c187cb4059599cabb328 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412099bd4979dd02b4d269cf58e8a365f2011_0fa6aa84c721c187cb4059599cabb328 2024-12-09T10:41:34,288 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209a17269eb3ba8421ba2e3489a31b97333_0fa6aa84c721c187cb4059599cabb328 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209a17269eb3ba8421ba2e3489a31b97333_0fa6aa84c721c187cb4059599cabb328 2024-12-09T10:41:34,289 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209acd8a996e28b4b7f90dd1b45524ded72_0fa6aa84c721c187cb4059599cabb328 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209acd8a996e28b4b7f90dd1b45524ded72_0fa6aa84c721c187cb4059599cabb328 2024-12-09T10:41:34,290 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209c552771bbcbf4b2192fcb1a4f948976b_0fa6aa84c721c187cb4059599cabb328 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209c552771bbcbf4b2192fcb1a4f948976b_0fa6aa84c721c187cb4059599cabb328 2024-12-09T10:41:34,291 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209c56833f921ee4edbbbae096026d02cdc_0fa6aa84c721c187cb4059599cabb328 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209c56833f921ee4edbbbae096026d02cdc_0fa6aa84c721c187cb4059599cabb328 2024-12-09T10:41:34,292 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209d566a027ff5a4b91b623c3156591ef32_0fa6aa84c721c187cb4059599cabb328 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209d566a027ff5a4b91b623c3156591ef32_0fa6aa84c721c187cb4059599cabb328 2024-12-09T10:41:34,293 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209ec7178558de34dcfa9a4a8825cc07e51_0fa6aa84c721c187cb4059599cabb328 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209ec7178558de34dcfa9a4a8825cc07e51_0fa6aa84c721c187cb4059599cabb328 2024-12-09T10:41:34,294 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209f236c4c878b84fc3bfb08d2c9e8fa9ba_0fa6aa84c721c187cb4059599cabb328 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209f236c4c878b84fc3bfb08d2c9e8fa9ba_0fa6aa84c721c187cb4059599cabb328 2024-12-09T10:41:34,295 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-09T10:41:34,297 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=56, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T10:41:34,299 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-09T10:41:34,301 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-09T10:41:34,302 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=56, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T10:41:34,302 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-09T10:41:34,303 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733740894303"}]},"ts":"9223372036854775807"} 2024-12-09T10:41:34,305 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-09T10:41:34,305 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 0fa6aa84c721c187cb4059599cabb328, NAME => 'TestAcidGuarantees,,1733740858386.0fa6aa84c721c187cb4059599cabb328.', STARTKEY => '', ENDKEY => ''}] 2024-12-09T10:41:34,305 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-09T10:41:34,305 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733740894305"}]},"ts":"9223372036854775807"} 2024-12-09T10:41:34,307 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-09T10:41:34,308 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=56, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T10:41:34,309 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 57 msec 2024-12-09T10:41:34,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-09T10:41:34,360 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-09T10:41:34,360 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-09T10:41:34,370 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=248 (was 249), OpenFileDescriptor=451 (was 457), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=418 (was 427), ProcessCount=11 (was 11), AvailableMemoryMB=8278 (was 8571) 2024-12-09T10:41:34,379 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=248, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=418, ProcessCount=11, AvailableMemoryMB=8277 2024-12-09T10:41:34,380 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-09T10:41:34,381 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T10:41:34,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] procedure2.ProcedureExecutor(1139): Stored pid=57, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-09T10:41:34,383 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-09T10:41:34,383 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:34,383 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 57 2024-12-09T10:41:34,384 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-09T10:41:34,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-12-09T10:41:34,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742062_1238 (size=1061) 2024-12-09T10:41:34,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-12-09T10:41:34,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-12-09T10:41:34,791 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160 2024-12-09T10:41:34,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742063_1239 (size=53) 2024-12-09T10:41:35,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-12-09T10:41:35,197 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T10:41:35,198 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing a9bc2b5e1505540f70456fbe90600f90, disabling compactions & flushes 2024-12-09T10:41:35,198 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:35,198 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:35,198 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. after waiting 0 ms 2024-12-09T10:41:35,198 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:35,198 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:35,198 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for a9bc2b5e1505540f70456fbe90600f90: Waiting for close lock at 1733740895198Disabling compacts and flushes for region at 1733740895198Disabling writes for close at 1733740895198Writing region close event to WAL at 1733740895198Closed at 1733740895198 2024-12-09T10:41:35,199 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-09T10:41:35,199 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733740895199"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733740895199"}]},"ts":"1733740895199"} 2024-12-09T10:41:35,201 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-09T10:41:35,201 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-09T10:41:35,202 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733740895201"}]},"ts":"1733740895201"} 2024-12-09T10:41:35,203 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-09T10:41:35,203 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a9bc2b5e1505540f70456fbe90600f90, ASSIGN}] 2024-12-09T10:41:35,204 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a9bc2b5e1505540f70456fbe90600f90, ASSIGN 2024-12-09T10:41:35,204 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a9bc2b5e1505540f70456fbe90600f90, ASSIGN; state=OFFLINE, location=7ea43b39fc24,36101,1733740827678; forceNewPlan=false, retain=false 2024-12-09T10:41:35,355 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=a9bc2b5e1505540f70456fbe90600f90, regionState=OPENING, regionLocation=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:35,357 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a9bc2b5e1505540f70456fbe90600f90, ASSIGN because future has completed 2024-12-09T10:41:35,357 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; OpenRegionProcedure a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678}] 2024-12-09T10:41:35,512 INFO [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:35,512 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7752): Opening region: {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} 2024-12-09T10:41:35,512 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees a9bc2b5e1505540f70456fbe90600f90 2024-12-09T10:41:35,512 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T10:41:35,512 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7794): checking encryption for a9bc2b5e1505540f70456fbe90600f90 2024-12-09T10:41:35,512 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7797): checking classloading for a9bc2b5e1505540f70456fbe90600f90 2024-12-09T10:41:35,514 INFO [StoreOpener-a9bc2b5e1505540f70456fbe90600f90-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region a9bc2b5e1505540f70456fbe90600f90 2024-12-09T10:41:35,515 INFO [StoreOpener-a9bc2b5e1505540f70456fbe90600f90-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T10:41:35,515 INFO [StoreOpener-a9bc2b5e1505540f70456fbe90600f90-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a9bc2b5e1505540f70456fbe90600f90 columnFamilyName A 2024-12-09T10:41:35,515 DEBUG [StoreOpener-a9bc2b5e1505540f70456fbe90600f90-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:35,515 INFO [StoreOpener-a9bc2b5e1505540f70456fbe90600f90-1 {}] regionserver.HStore(327): Store=a9bc2b5e1505540f70456fbe90600f90/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T10:41:35,516 INFO [StoreOpener-a9bc2b5e1505540f70456fbe90600f90-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region a9bc2b5e1505540f70456fbe90600f90 2024-12-09T10:41:35,516 INFO [StoreOpener-a9bc2b5e1505540f70456fbe90600f90-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T10:41:35,517 INFO [StoreOpener-a9bc2b5e1505540f70456fbe90600f90-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a9bc2b5e1505540f70456fbe90600f90 columnFamilyName B 2024-12-09T10:41:35,517 DEBUG [StoreOpener-a9bc2b5e1505540f70456fbe90600f90-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:35,517 INFO [StoreOpener-a9bc2b5e1505540f70456fbe90600f90-1 {}] regionserver.HStore(327): Store=a9bc2b5e1505540f70456fbe90600f90/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T10:41:35,517 INFO [StoreOpener-a9bc2b5e1505540f70456fbe90600f90-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region a9bc2b5e1505540f70456fbe90600f90 2024-12-09T10:41:35,518 INFO [StoreOpener-a9bc2b5e1505540f70456fbe90600f90-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T10:41:35,518 INFO [StoreOpener-a9bc2b5e1505540f70456fbe90600f90-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a9bc2b5e1505540f70456fbe90600f90 columnFamilyName C 2024-12-09T10:41:35,518 DEBUG [StoreOpener-a9bc2b5e1505540f70456fbe90600f90-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:41:35,518 INFO [StoreOpener-a9bc2b5e1505540f70456fbe90600f90-1 {}] regionserver.HStore(327): Store=a9bc2b5e1505540f70456fbe90600f90/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T10:41:35,518 INFO [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:35,518 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1038): replaying wal for a9bc2b5e1505540f70456fbe90600f90 2024-12-09T10:41:35,519 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90 2024-12-09T10:41:35,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-12-09T10:41:35,519 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90 2024-12-09T10:41:35,520 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1048): stopping wal replay for a9bc2b5e1505540f70456fbe90600f90 2024-12-09T10:41:35,520 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1060): Cleaning up temporary data for a9bc2b5e1505540f70456fbe90600f90 2024-12-09T10:41:35,521 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-09T10:41:35,522 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1093): writing seq id for a9bc2b5e1505540f70456fbe90600f90 2024-12-09T10:41:35,524 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T10:41:35,524 INFO [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1114): Opened a9bc2b5e1505540f70456fbe90600f90; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70300611, jitterRate=0.047560736536979675}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-09T10:41:35,524 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a9bc2b5e1505540f70456fbe90600f90 2024-12-09T10:41:35,525 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1006): Region open journal for a9bc2b5e1505540f70456fbe90600f90: Running coprocessor pre-open hook at 1733740895512Writing region info on filesystem at 1733740895512Initializing all the Stores at 1733740895513 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733740895513Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733740895513Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733740895513Cleaning up temporary data from old regions at 1733740895520 (+7 ms)Running coprocessor post-open hooks at 1733740895524 (+4 ms)Region opened successfully at 1733740895525 (+1 ms) 2024-12-09T10:41:35,526 INFO [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., pid=59, masterSystemTime=1733740895509 2024-12-09T10:41:35,527 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:35,527 INFO [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:35,528 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=a9bc2b5e1505540f70456fbe90600f90, regionState=OPEN, openSeqNum=2, regionLocation=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:35,529 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=59, ppid=58, state=RUNNABLE, hasLock=false; OpenRegionProcedure a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 because future has completed 2024-12-09T10:41:35,531 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-12-09T10:41:35,531 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; OpenRegionProcedure a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 in 173 msec 2024-12-09T10:41:35,533 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-12-09T10:41:35,533 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a9bc2b5e1505540f70456fbe90600f90, ASSIGN in 328 msec 2024-12-09T10:41:35,533 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-09T10:41:35,533 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733740895533"}]},"ts":"1733740895533"} 2024-12-09T10:41:35,535 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-09T10:41:35,535 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-09T10:41:35,537 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1540 sec 2024-12-09T10:41:36,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-12-09T10:41:36,530 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-09T10:41:36,531 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6eecc019, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:41:36,531 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ea43b39fc24,35095,-1 for getting cluster id 2024-12-09T10:41:36,531 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T10:41:36,532 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0945b5ae-b46b-4b8f-af48-56b0bb92df09' 2024-12-09T10:41:36,532 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T10:41:36,532 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0945b5ae-b46b-4b8f-af48-56b0bb92df09" 2024-12-09T10:41:36,532 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@166a1c71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:41:36,532 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ea43b39fc24,35095,-1] 2024-12-09T10:41:36,533 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T10:41:36,533 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:41:36,534 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35372, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T10:41:36,534 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4816cc63, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:41:36,534 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T10:41:36,535 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ea43b39fc24,36101,1733740827678, seqNum=-1] 2024-12-09T10:41:36,535 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T10:41:36,536 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50330, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T10:41:36,537 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-09T10:41:36,538 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 7ea43b39fc24,35095,1733740826918 2024-12-09T10:41:36,538 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7f1a8e17 2024-12-09T10:41:36,538 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-09T10:41:36,538 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35380, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-09T10:41:36,539 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@cbf7aae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:41:36,540 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ea43b39fc24,35095,-1 for getting cluster id 2024-12-09T10:41:36,540 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T10:41:36,540 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0945b5ae-b46b-4b8f-af48-56b0bb92df09' 2024-12-09T10:41:36,540 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T10:41:36,540 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0945b5ae-b46b-4b8f-af48-56b0bb92df09" 2024-12-09T10:41:36,540 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cd44a3e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:41:36,540 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ea43b39fc24,35095,-1] 2024-12-09T10:41:36,541 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T10:41:36,541 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:41:36,541 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35404, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T10:41:36,542 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6794b2be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:41:36,542 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43be4cb6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:41:36,542 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ea43b39fc24,35095,-1 for getting cluster id 2024-12-09T10:41:36,542 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T10:41:36,543 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0945b5ae-b46b-4b8f-af48-56b0bb92df09' 2024-12-09T10:41:36,543 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T10:41:36,543 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0945b5ae-b46b-4b8f-af48-56b0bb92df09" 2024-12-09T10:41:36,543 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@685c3cb9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:41:36,543 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ea43b39fc24,35095,-1] 2024-12-09T10:41:36,543 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T10:41:36,544 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:41:36,544 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35430, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T10:41:36,545 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@bd7968c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:41:36,545 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ff85c2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:41:36,545 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ea43b39fc24,35095,-1 for getting cluster id 2024-12-09T10:41:36,545 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T10:41:36,546 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0945b5ae-b46b-4b8f-af48-56b0bb92df09' 2024-12-09T10:41:36,546 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T10:41:36,546 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0945b5ae-b46b-4b8f-af48-56b0bb92df09" 2024-12-09T10:41:36,546 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c63978f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:41:36,546 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ea43b39fc24,35095,-1] 2024-12-09T10:41:36,546 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T10:41:36,546 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:41:36,547 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35448, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T10:41:36,547 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4513646a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:41:36,548 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c504a5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:41:36,548 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ea43b39fc24,35095,-1 for getting cluster id 2024-12-09T10:41:36,548 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T10:41:36,548 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0945b5ae-b46b-4b8f-af48-56b0bb92df09' 2024-12-09T10:41:36,548 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T10:41:36,548 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0945b5ae-b46b-4b8f-af48-56b0bb92df09" 2024-12-09T10:41:36,548 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2dce71db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:41:36,548 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ea43b39fc24,35095,-1] 2024-12-09T10:41:36,549 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T10:41:36,549 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:41:36,549 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35464, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T10:41:36,550 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45abe4d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:41:36,550 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1546013f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:41:36,550 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ea43b39fc24,35095,-1 for getting cluster id 2024-12-09T10:41:36,550 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T10:41:36,551 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0945b5ae-b46b-4b8f-af48-56b0bb92df09' 2024-12-09T10:41:36,551 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T10:41:36,551 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0945b5ae-b46b-4b8f-af48-56b0bb92df09" 2024-12-09T10:41:36,551 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32d85bbb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:41:36,551 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ea43b39fc24,35095,-1] 2024-12-09T10:41:36,551 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T10:41:36,551 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:41:36,552 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35486, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T10:41:36,552 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@346d7039, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:41:36,553 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@163827fe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:41:36,553 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ea43b39fc24,35095,-1 for getting cluster id 2024-12-09T10:41:36,553 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T10:41:36,554 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0945b5ae-b46b-4b8f-af48-56b0bb92df09' 2024-12-09T10:41:36,554 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T10:41:36,554 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0945b5ae-b46b-4b8f-af48-56b0bb92df09" 2024-12-09T10:41:36,554 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2987dc6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:41:36,554 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ea43b39fc24,35095,-1] 2024-12-09T10:41:36,554 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T10:41:36,554 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:41:36,555 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35508, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T10:41:36,555 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69ddffbc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:41:36,556 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28aafd78, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:41:36,556 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ea43b39fc24,35095,-1 for getting cluster id 2024-12-09T10:41:36,556 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T10:41:36,556 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0945b5ae-b46b-4b8f-af48-56b0bb92df09' 2024-12-09T10:41:36,557 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T10:41:36,557 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0945b5ae-b46b-4b8f-af48-56b0bb92df09" 2024-12-09T10:41:36,557 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40ad893d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:41:36,557 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ea43b39fc24,35095,-1] 2024-12-09T10:41:36,557 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T10:41:36,557 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:41:36,558 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35532, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T10:41:36,558 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@122cdf2e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:41:36,558 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f4d0c90, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:41:36,558 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ea43b39fc24,35095,-1 for getting cluster id 2024-12-09T10:41:36,558 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T10:41:36,559 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0945b5ae-b46b-4b8f-af48-56b0bb92df09' 2024-12-09T10:41:36,559 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T10:41:36,559 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0945b5ae-b46b-4b8f-af48-56b0bb92df09" 2024-12-09T10:41:36,559 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78455341, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:41:36,559 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ea43b39fc24,35095,-1] 2024-12-09T10:41:36,559 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T10:41:36,560 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:41:36,560 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35554, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T10:41:36,560 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73c343e3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:41:36,561 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30a8ae92, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:41:36,561 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ea43b39fc24,35095,-1 for getting cluster id 2024-12-09T10:41:36,561 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T10:41:36,562 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0945b5ae-b46b-4b8f-af48-56b0bb92df09' 2024-12-09T10:41:36,562 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T10:41:36,562 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0945b5ae-b46b-4b8f-af48-56b0bb92df09" 2024-12-09T10:41:36,562 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@509c3c97, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:41:36,562 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ea43b39fc24,35095,-1] 2024-12-09T10:41:36,562 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T10:41:36,562 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:41:36,563 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35580, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T10:41:36,563 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6734d07e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:41:36,564 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22d18cdc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:41:36,564 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ea43b39fc24,35095,-1 for getting cluster id 2024-12-09T10:41:36,564 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T10:41:36,564 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0945b5ae-b46b-4b8f-af48-56b0bb92df09' 2024-12-09T10:41:36,564 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T10:41:36,564 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0945b5ae-b46b-4b8f-af48-56b0bb92df09" 2024-12-09T10:41:36,565 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b98a4f3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:41:36,565 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ea43b39fc24,35095,-1] 2024-12-09T10:41:36,565 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T10:41:36,565 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:41:36,565 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35604, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T10:41:36,566 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@441f821c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:41:36,566 DEBUG [Thread-1121 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T10:41:36,566 DEBUG [Thread-1130 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T10:41:36,566 DEBUG [Thread-1132 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T10:41:36,566 DEBUG [Thread-1134 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T10:41:36,567 DEBUG [Thread-1126 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T10:41:36,567 DEBUG [Thread-1117 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T10:41:36,567 DEBUG [Thread-1128 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T10:41:36,567 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ea43b39fc24,36101,1733740827678, seqNum=-1] 2024-12-09T10:41:36,567 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ea43b39fc24,36101,1733740827678, seqNum=-1] 2024-12-09T10:41:36,567 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ea43b39fc24,36101,1733740827678, seqNum=-1] 2024-12-09T10:41:36,567 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ea43b39fc24,36101,1733740827678, seqNum=-1] 2024-12-09T10:41:36,567 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ea43b39fc24,36101,1733740827678, seqNum=-1] 2024-12-09T10:41:36,567 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T10:41:36,567 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T10:41:36,567 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T10:41:36,567 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T10:41:36,568 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T10:41:36,568 DEBUG [Thread-1115 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T10:41:36,568 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ea43b39fc24,36101,1733740827678, seqNum=-1] 2024-12-09T10:41:36,569 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50362, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T10:41:36,569 DEBUG [Thread-1119 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T10:41:36,569 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50346, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T10:41:36,569 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T10:41:36,569 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50378, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T10:41:36,569 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ea43b39fc24,36101,1733740827678, seqNum=-1] 2024-12-09T10:41:36,569 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T10:41:36,569 DEBUG [Thread-1123 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T10:41:36,570 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50340, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T10:41:36,570 DEBUG [Thread-1115 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ea43b39fc24,36101,1733740827678, seqNum=-1] 2024-12-09T10:41:36,570 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50390, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T10:41:36,570 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50364, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T10:41:36,570 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ea43b39fc24,36101,1733740827678, seqNum=-1] 2024-12-09T10:41:36,570 DEBUG [Thread-1115 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T10:41:36,570 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-09T10:41:36,570 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T10:41:36,570 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50404, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T10:41:36,571 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2] 2024-12-09T10:41:36,571 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50408, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T10:41:36,571 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50410, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T10:41:36,572 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2] 2024-12-09T10:41:36,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees 2024-12-09T10:41:36,572 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ea43b39fc24,36101,1733740827678, seqNum=-1] 2024-12-09T10:41:36,572 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2] 2024-12-09T10:41:36,573 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T10:41:36,573 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2] 2024-12-09T10:41:36,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-09T10:41:36,574 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T10:41:36,574 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2] 2024-12-09T10:41:36,575 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2] 2024-12-09T10:41:36,575 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2] 2024-12-09T10:41:36,575 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2] 2024-12-09T10:41:36,575 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50422, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T10:41:36,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2] 2024-12-09T10:41:36,576 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T10:41:36,576 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T10:41:36,577 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2] 2024-12-09T10:41:36,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(8855): Flush requested on a9bc2b5e1505540f70456fbe90600f90 2024-12-09T10:41:36,584 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a9bc2b5e1505540f70456fbe90600f90 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-09T10:41:36,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9bc2b5e1505540f70456fbe90600f90, store=A 2024-12-09T10:41:36,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:41:36,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9bc2b5e1505540f70456fbe90600f90, store=B 2024-12-09T10:41:36,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:41:36,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9bc2b5e1505540f70456fbe90600f90, store=C 2024-12-09T10:41:36,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:41:36,618 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/A/c151e1a164ac49ab93c3ee4a6e0dad59 is 50, key is test_row_0/A:col10/1733740896583/Put/seqid=0 2024-12-09T10:41:36,621 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:36,621 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:36,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50340 deadline: 1733740956614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:36,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50410 deadline: 1733740956612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:36,622 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:36,622 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:36,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50422 deadline: 1733740956615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:36,622 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:36,622 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:36,622 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:36,622 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:36,622 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:36,622 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:36,622 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:36,622 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:36,624 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:36,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50408 deadline: 1733740956617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:36,624 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:36,624 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:36,624 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:36,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50390 deadline: 1733740956618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:36,624 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:36,625 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:36,625 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:36,625 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:36,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742064_1240 (size=16681) 2024-12-09T10:41:36,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-09T10:41:36,732 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:36,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50410 deadline: 1733740956730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:36,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:36,733 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:36,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50422 deadline: 1733740956730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:36,733 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:36,733 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:36,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:36,733 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:36,733 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:36,733 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:36,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50340 deadline: 1733740956730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:36,733 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:36,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50408 deadline: 1733740956730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:36,733 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:36,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:36,733 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:36,734 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:36,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50390 deadline: 1733740956730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:36,734 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:36,734 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:36,734 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:36,734 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:36,734 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:36,734 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:36,738 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-12-09T10:41:36,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:36,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. as already flushing 2024-12-09T10:41:36,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:36,738 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:36,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:36,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:36,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-09T10:41:36,891 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-12-09T10:41:36,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:36,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. as already flushing 2024-12-09T10:41:36,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:36,892 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:36,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:36,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:36,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:36,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50410 deadline: 1733740956940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:36,942 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:36,942 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:36,942 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:36,942 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:36,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50422 deadline: 1733740956940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:36,942 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:36,942 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:36,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50340 deadline: 1733740956940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:36,942 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:36,942 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:36,942 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:36,942 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:36,943 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:36,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50408 deadline: 1733740956940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:36,943 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:36,943 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:36,943 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:36,943 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:36,943 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:36,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50390 deadline: 1733740956940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:36,943 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:36,943 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:36,943 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:37,036 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/A/c151e1a164ac49ab93c3ee4a6e0dad59 2024-12-09T10:41:37,045 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-12-09T10:41:37,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:37,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. as already flushing 2024-12-09T10:41:37,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:37,045 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:37,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:37,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:37,065 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/B/0534f3f0656840f494cf63787da7e735 is 50, key is test_row_0/B:col10/1733740896583/Put/seqid=0 2024-12-09T10:41:37,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742065_1241 (size=12001) 2024-12-09T10:41:37,072 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/B/0534f3f0656840f494cf63787da7e735 2024-12-09T10:41:37,097 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/C/0332bb2996bb402d8e2e02dd6da0bef0 is 50, key is test_row_0/C:col10/1733740896583/Put/seqid=0 2024-12-09T10:41:37,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742066_1242 (size=12001) 2024-12-09T10:41:37,199 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-12-09T10:41:37,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:37,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. as already flushing 2024-12-09T10:41:37,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:37,200 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:37,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:37,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:37,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-09T10:41:37,252 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:37,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50410 deadline: 1733740957250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:37,253 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:37,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50422 deadline: 1733740957250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:37,253 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:37,253 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:37,253 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:37,253 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:37,253 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:37,253 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:37,253 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:37,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50340 deadline: 1733740957250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:37,253 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:37,253 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:37,253 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:37,253 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:37,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50408 deadline: 1733740957250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:37,253 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:37,254 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:37,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50390 deadline: 1733740957250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:37,254 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:37,254 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:37,254 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:37,254 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:37,254 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:37,353 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-12-09T10:41:37,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:37,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. as already flushing 2024-12-09T10:41:37,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:37,354 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:37,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:37,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:37,504 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/C/0332bb2996bb402d8e2e02dd6da0bef0 2024-12-09T10:41:37,506 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-12-09T10:41:37,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:37,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. as already flushing 2024-12-09T10:41:37,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:37,507 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:37,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:37,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:37,509 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/A/c151e1a164ac49ab93c3ee4a6e0dad59 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/c151e1a164ac49ab93c3ee4a6e0dad59 2024-12-09T10:41:37,513 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/c151e1a164ac49ab93c3ee4a6e0dad59, entries=250, sequenceid=14, filesize=16.3 K 2024-12-09T10:41:37,514 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/B/0534f3f0656840f494cf63787da7e735 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/0534f3f0656840f494cf63787da7e735 2024-12-09T10:41:37,518 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/0534f3f0656840f494cf63787da7e735, entries=150, sequenceid=14, filesize=11.7 K 2024-12-09T10:41:37,520 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/C/0332bb2996bb402d8e2e02dd6da0bef0 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/0332bb2996bb402d8e2e02dd6da0bef0 2024-12-09T10:41:37,525 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/0332bb2996bb402d8e2e02dd6da0bef0, entries=150, sequenceid=14, filesize=11.7 K 2024-12-09T10:41:37,526 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for a9bc2b5e1505540f70456fbe90600f90 in 942ms, sequenceid=14, compaction requested=false 2024-12-09T10:41:37,526 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a9bc2b5e1505540f70456fbe90600f90: 2024-12-09T10:41:37,660 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-12-09T10:41:37,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:37,661 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2902): Flushing a9bc2b5e1505540f70456fbe90600f90 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-09T10:41:37,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9bc2b5e1505540f70456fbe90600f90, store=A 2024-12-09T10:41:37,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:41:37,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9bc2b5e1505540f70456fbe90600f90, store=B 2024-12-09T10:41:37,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:41:37,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9bc2b5e1505540f70456fbe90600f90, store=C 2024-12-09T10:41:37,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:41:37,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/A/83d14cd85d61439c8ac3df3995caa7d2 is 50, key is test_row_0/A:col10/1733740896615/Put/seqid=0 2024-12-09T10:41:37,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742067_1243 (size=12001) 2024-12-09T10:41:37,694 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=34 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/A/83d14cd85d61439c8ac3df3995caa7d2 2024-12-09T10:41:37,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/B/42a64d85a8c94b1a95a1d75281a0c2ad is 50, key is test_row_0/B:col10/1733740896615/Put/seqid=0 2024-12-09T10:41:37,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-09T10:41:37,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742068_1244 (size=12001) 2024-12-09T10:41:37,716 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=34 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/B/42a64d85a8c94b1a95a1d75281a0c2ad 2024-12-09T10:41:37,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/C/dc781da867f4487ca634b88ddadff9dd is 50, key is test_row_0/C:col10/1733740896615/Put/seqid=0 2024-12-09T10:41:37,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742069_1245 (size=12001) 2024-12-09T10:41:37,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(8855): Flush requested on a9bc2b5e1505540f70456fbe90600f90 2024-12-09T10:41:37,762 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. as already flushing 2024-12-09T10:41:37,772 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:37,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50340 deadline: 1733740957764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:37,773 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:37,773 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:37,773 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:37,777 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:37,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50410 deadline: 1733740957770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:37,777 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:37,777 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:37,777 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:37,777 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:37,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50408 deadline: 1733740957771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:37,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50422 deadline: 1733740957770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:37,777 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:37,778 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:37,778 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:37,778 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:37,778 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:37,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50390 deadline: 1733740957772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:37,778 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:37,778 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:37,778 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:37,778 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:37,778 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:37,778 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:37,882 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:37,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50340 deadline: 1733740957880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:37,883 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:37,883 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:37,883 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:37,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50410 deadline: 1733740957880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:37,883 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:37,883 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:37,883 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:37,883 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:37,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50408 deadline: 1733740957880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:37,883 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:37,883 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:37,883 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:37,883 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:37,883 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:37,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50422 deadline: 1733740957880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:37,884 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:37,884 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:37,884 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:37,892 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:37,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50390 deadline: 1733740957889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:37,892 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:37,892 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:37,892 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:38,090 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:38,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50340 deadline: 1733740958089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:38,091 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:38,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50410 deadline: 1733740958089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:38,091 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:38,091 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:38,091 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:38,091 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:38,091 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:38,091 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:38,093 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:38,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50408 deadline: 1733740958089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:38,093 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:38,093 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:38,093 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:38,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50422 deadline: 1733740958090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:38,093 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:38,094 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:38,094 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:38,094 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:38,103 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:38,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50390 deadline: 1733740958100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:38,103 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:38,104 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:38,104 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:38,134 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=34 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/C/dc781da867f4487ca634b88ddadff9dd 2024-12-09T10:41:38,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/A/83d14cd85d61439c8ac3df3995caa7d2 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/83d14cd85d61439c8ac3df3995caa7d2 2024-12-09T10:41:38,144 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/83d14cd85d61439c8ac3df3995caa7d2, entries=150, sequenceid=34, filesize=11.7 K 2024-12-09T10:41:38,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/B/42a64d85a8c94b1a95a1d75281a0c2ad as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/42a64d85a8c94b1a95a1d75281a0c2ad 2024-12-09T10:41:38,149 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/42a64d85a8c94b1a95a1d75281a0c2ad, entries=150, sequenceid=34, filesize=11.7 K 2024-12-09T10:41:38,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/C/dc781da867f4487ca634b88ddadff9dd as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/dc781da867f4487ca634b88ddadff9dd 2024-12-09T10:41:38,155 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/dc781da867f4487ca634b88ddadff9dd, entries=150, sequenceid=34, filesize=11.7 K 2024-12-09T10:41:38,156 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for a9bc2b5e1505540f70456fbe90600f90 in 496ms, sequenceid=34, compaction requested=false 2024-12-09T10:41:38,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2603): Flush status journal for a9bc2b5e1505540f70456fbe90600f90: 2024-12-09T10:41:38,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:38,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=61 2024-12-09T10:41:38,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4169): Remote procedure done, pid=61 2024-12-09T10:41:38,160 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-12-09T10:41:38,160 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5820 sec 2024-12-09T10:41:38,164 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees in 1.5910 sec 2024-12-09T10:41:38,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(8855): Flush requested on a9bc2b5e1505540f70456fbe90600f90 2024-12-09T10:41:38,402 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a9bc2b5e1505540f70456fbe90600f90 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-09T10:41:38,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9bc2b5e1505540f70456fbe90600f90, store=A 2024-12-09T10:41:38,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:41:38,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9bc2b5e1505540f70456fbe90600f90, store=B 2024-12-09T10:41:38,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:41:38,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9bc2b5e1505540f70456fbe90600f90, store=C 2024-12-09T10:41:38,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:41:38,410 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/A/7e065977abbf4c5aa2eb93592b62d999 is 50, key is test_row_0/A:col10/1733740898402/Put/seqid=0 2024-12-09T10:41:38,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742070_1246 (size=14341) 2024-12-09T10:41:38,425 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:38,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50422 deadline: 1733740958418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:38,426 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:38,426 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:38,426 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:38,426 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:38,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50410 deadline: 1733740958423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:38,426 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:38,426 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:38,426 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:38,430 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:38,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50390 deadline: 1733740958424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:38,431 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:38,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50408 deadline: 1733740958424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:38,431 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:38,431 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:38,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:38,431 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:38,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:38,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50340 deadline: 1733740958425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:38,431 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:38,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:38,432 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:38,432 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:38,432 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:38,533 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:38,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50422 deadline: 1733740958530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:38,533 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:38,533 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:38,533 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:38,533 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:38,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50410 deadline: 1733740958530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:38,534 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:38,534 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:38,534 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:38,543 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:38,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50390 deadline: 1733740958539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:38,543 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:38,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50408 deadline: 1733740958539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:38,543 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:38,543 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:38,543 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:38,543 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:38,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50340 deadline: 1733740958539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:38,544 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:38,544 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:38,544 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:38,544 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:38,544 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:38,544 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:38,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-09T10:41:38,719 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T10:41:38,720 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-09T10:41:38,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees 2024-12-09T10:41:38,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-09T10:41:38,722 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T10:41:38,722 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T10:41:38,722 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T10:41:38,742 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:38,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50422 deadline: 1733740958739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:38,743 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:38,743 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:38,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50410 deadline: 1733740958739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:38,743 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:38,743 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:38,743 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:38,743 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:38,743 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:38,752 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:38,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50390 deadline: 1733740958750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:38,753 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:38,753 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:38,753 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:38,753 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:38,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50408 deadline: 1733740958750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:38,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:38,753 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:38,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:38,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:38,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50340 deadline: 1733740958750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:38,754 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:38,754 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:38,754 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:38,815 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/A/7e065977abbf4c5aa2eb93592b62d999 2024-12-09T10:41:38,823 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/B/39a8862ba4e742c182751de99eb3dcd2 is 50, key is test_row_0/B:col10/1733740898402/Put/seqid=0 2024-12-09T10:41:38,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742071_1247 (size=12001) 2024-12-09T10:41:38,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-09T10:41:38,874 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-09T10:41:38,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:38,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. as already flushing 2024-12-09T10:41:38,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:38,875 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:38,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:38,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:39,028 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-09T10:41:39,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:39,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. as already flushing 2024-12-09T10:41:39,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:39,028 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:39,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:39,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:39,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-09T10:41:39,052 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:39,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50422 deadline: 1733740959049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:39,053 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:39,053 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:39,053 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:39,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50410 deadline: 1733740959050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:39,053 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:39,053 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:39,053 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:39,053 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:39,062 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:39,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50390 deadline: 1733740959060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:39,062 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:39,062 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:39,062 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:39,062 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:39,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50340 deadline: 1733740959060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:39,062 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:39,062 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:39,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50408 deadline: 1733740959060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:39,062 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:39,062 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:39,063 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:39,063 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:39,063 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:39,182 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-09T10:41:39,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:39,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. as already flushing 2024-12-09T10:41:39,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:39,182 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:39,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:39,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:39,228 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/B/39a8862ba4e742c182751de99eb3dcd2 2024-12-09T10:41:39,236 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/C/57ba7a2a58c54352b1eed8458309ab5e is 50, key is test_row_0/C:col10/1733740898402/Put/seqid=0 2024-12-09T10:41:39,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742072_1248 (size=12001) 2024-12-09T10:41:39,335 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-09T10:41:39,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:39,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. as already flushing 2024-12-09T10:41:39,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:39,335 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:39,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:39,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:39,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-09T10:41:39,488 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-09T10:41:39,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:39,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. as already flushing 2024-12-09T10:41:39,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:39,488 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:39,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:39,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:39,534 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-09T10:41:39,560 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:39,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50410 deadline: 1733740959560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:39,561 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:39,561 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:39,561 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:39,561 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:39,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50422 deadline: 1733740959560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:39,561 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:39,561 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:39,562 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:39,572 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:39,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50390 deadline: 1733740959569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:39,572 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:39,572 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:39,572 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:39,572 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:39,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50408 deadline: 1733740959570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:39,573 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:39,573 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:39,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50340 deadline: 1733740959570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:39,573 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:39,573 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:39,573 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:39,573 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:39,573 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:39,640 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/C/57ba7a2a58c54352b1eed8458309ab5e 2024-12-09T10:41:39,641 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-09T10:41:39,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:39,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. as already flushing 2024-12-09T10:41:39,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:39,641 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:39,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:39,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:39,646 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/A/7e065977abbf4c5aa2eb93592b62d999 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/7e065977abbf4c5aa2eb93592b62d999 2024-12-09T10:41:39,650 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/7e065977abbf4c5aa2eb93592b62d999, entries=200, sequenceid=49, filesize=14.0 K 2024-12-09T10:41:39,651 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/B/39a8862ba4e742c182751de99eb3dcd2 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/39a8862ba4e742c182751de99eb3dcd2 2024-12-09T10:41:39,655 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/39a8862ba4e742c182751de99eb3dcd2, entries=150, sequenceid=49, filesize=11.7 K 2024-12-09T10:41:39,656 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/C/57ba7a2a58c54352b1eed8458309ab5e as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/57ba7a2a58c54352b1eed8458309ab5e 2024-12-09T10:41:39,660 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/57ba7a2a58c54352b1eed8458309ab5e, entries=150, sequenceid=49, filesize=11.7 K 2024-12-09T10:41:39,661 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for a9bc2b5e1505540f70456fbe90600f90 in 1259ms, sequenceid=49, compaction requested=true 2024-12-09T10:41:39,661 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a9bc2b5e1505540f70456fbe90600f90: 2024-12-09T10:41:39,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9bc2b5e1505540f70456fbe90600f90:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T10:41:39,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:41:39,661 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T10:41:39,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9bc2b5e1505540f70456fbe90600f90:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T10:41:39,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:41:39,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9bc2b5e1505540f70456fbe90600f90:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T10:41:39,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T10:41:39,661 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T10:41:39,663 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T10:41:39,663 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1541): a9bc2b5e1505540f70456fbe90600f90/B is initiating minor compaction (all files) 2024-12-09T10:41:39,663 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a9bc2b5e1505540f70456fbe90600f90/B in TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:39,663 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/0534f3f0656840f494cf63787da7e735, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/42a64d85a8c94b1a95a1d75281a0c2ad, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/39a8862ba4e742c182751de99eb3dcd2] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp, totalSize=35.2 K 2024-12-09T10:41:39,663 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43023 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T10:41:39,663 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1541): a9bc2b5e1505540f70456fbe90600f90/A is initiating minor compaction (all files) 2024-12-09T10:41:39,663 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a9bc2b5e1505540f70456fbe90600f90/A in TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:39,663 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/c151e1a164ac49ab93c3ee4a6e0dad59, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/83d14cd85d61439c8ac3df3995caa7d2, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/7e065977abbf4c5aa2eb93592b62d999] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp, totalSize=42.0 K 2024-12-09T10:41:39,664 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 0534f3f0656840f494cf63787da7e735, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733740896580 2024-12-09T10:41:39,664 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting c151e1a164ac49ab93c3ee4a6e0dad59, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733740896575 2024-12-09T10:41:39,664 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 42a64d85a8c94b1a95a1d75281a0c2ad, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=34, earliestPutTs=1733740896600 2024-12-09T10:41:39,664 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 83d14cd85d61439c8ac3df3995caa7d2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=34, earliestPutTs=1733740896600 2024-12-09T10:41:39,665 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7e065977abbf4c5aa2eb93592b62d999, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733740897771 2024-12-09T10:41:39,665 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 39a8862ba4e742c182751de99eb3dcd2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733740898401 2024-12-09T10:41:39,712 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a9bc2b5e1505540f70456fbe90600f90#B#compaction#204 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:41:39,713 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/B/7ffe5f63c06846bea7fb035e9a61eaf7 is 50, key is test_row_0/B:col10/1733740898402/Put/seqid=0 2024-12-09T10:41:39,716 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a9bc2b5e1505540f70456fbe90600f90#A#compaction#205 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:41:39,716 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/A/41528c70c5c4471084f0037919b20b35 is 50, key is test_row_0/A:col10/1733740898402/Put/seqid=0 2024-12-09T10:41:39,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742074_1250 (size=12104) 2024-12-09T10:41:39,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742073_1249 (size=12104) 2024-12-09T10:41:39,752 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/B/7ffe5f63c06846bea7fb035e9a61eaf7 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/7ffe5f63c06846bea7fb035e9a61eaf7 2024-12-09T10:41:39,758 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a9bc2b5e1505540f70456fbe90600f90/B of a9bc2b5e1505540f70456fbe90600f90 into 7ffe5f63c06846bea7fb035e9a61eaf7(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:41:39,758 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a9bc2b5e1505540f70456fbe90600f90: 2024-12-09T10:41:39,758 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., storeName=a9bc2b5e1505540f70456fbe90600f90/B, priority=13, startTime=1733740899661; duration=0sec 2024-12-09T10:41:39,758 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T10:41:39,758 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9bc2b5e1505540f70456fbe90600f90:B 2024-12-09T10:41:39,758 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T10:41:39,759 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T10:41:39,759 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1541): a9bc2b5e1505540f70456fbe90600f90/C is initiating minor compaction (all files) 2024-12-09T10:41:39,760 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a9bc2b5e1505540f70456fbe90600f90/C in TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:39,760 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/0332bb2996bb402d8e2e02dd6da0bef0, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/dc781da867f4487ca634b88ddadff9dd, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/57ba7a2a58c54352b1eed8458309ab5e] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp, totalSize=35.2 K 2024-12-09T10:41:39,760 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 0332bb2996bb402d8e2e02dd6da0bef0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733740896580 2024-12-09T10:41:39,760 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting dc781da867f4487ca634b88ddadff9dd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=34, earliestPutTs=1733740896600 2024-12-09T10:41:39,761 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 57ba7a2a58c54352b1eed8458309ab5e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733740898401 2024-12-09T10:41:39,769 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a9bc2b5e1505540f70456fbe90600f90#C#compaction#206 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:41:39,769 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/C/9d2489362d24431ba9bd18c40d9b2f8d is 50, key is test_row_0/C:col10/1733740898402/Put/seqid=0 2024-12-09T10:41:39,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742075_1251 (size=12104) 2024-12-09T10:41:39,785 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/C/9d2489362d24431ba9bd18c40d9b2f8d as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/9d2489362d24431ba9bd18c40d9b2f8d 2024-12-09T10:41:39,790 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a9bc2b5e1505540f70456fbe90600f90/C of a9bc2b5e1505540f70456fbe90600f90 into 9d2489362d24431ba9bd18c40d9b2f8d(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:41:39,790 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a9bc2b5e1505540f70456fbe90600f90: 2024-12-09T10:41:39,790 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., storeName=a9bc2b5e1505540f70456fbe90600f90/C, priority=13, startTime=1733740899661; duration=0sec 2024-12-09T10:41:39,790 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:41:39,790 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9bc2b5e1505540f70456fbe90600f90:C 2024-12-09T10:41:39,794 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-09T10:41:39,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:39,795 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2902): Flushing a9bc2b5e1505540f70456fbe90600f90 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-09T10:41:39,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9bc2b5e1505540f70456fbe90600f90, store=A 2024-12-09T10:41:39,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:41:39,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9bc2b5e1505540f70456fbe90600f90, store=B 2024-12-09T10:41:39,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:41:39,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9bc2b5e1505540f70456fbe90600f90, store=C 2024-12-09T10:41:39,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:41:39,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/A/b8e20005785443c58e347a18f9f8d8bf is 50, key is test_row_0/A:col10/1733740898421/Put/seqid=0 2024-12-09T10:41:39,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742076_1252 (size=12001) 2024-12-09T10:41:39,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-09T10:41:40,153 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/A/41528c70c5c4471084f0037919b20b35 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/41528c70c5c4471084f0037919b20b35 2024-12-09T10:41:40,161 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a9bc2b5e1505540f70456fbe90600f90/A of a9bc2b5e1505540f70456fbe90600f90 into 41528c70c5c4471084f0037919b20b35(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:41:40,162 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a9bc2b5e1505540f70456fbe90600f90: 2024-12-09T10:41:40,162 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., storeName=a9bc2b5e1505540f70456fbe90600f90/A, priority=13, startTime=1733740899661; duration=0sec 2024-12-09T10:41:40,162 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:41:40,162 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9bc2b5e1505540f70456fbe90600f90:A 2024-12-09T10:41:40,216 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=68 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/A/b8e20005785443c58e347a18f9f8d8bf 2024-12-09T10:41:40,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/B/0d23e59a4089427e9895a130af9c4713 is 50, key is test_row_0/B:col10/1733740898421/Put/seqid=0 2024-12-09T10:41:40,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742077_1253 (size=12001) 2024-12-09T10:41:40,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(8855): Flush requested on a9bc2b5e1505540f70456fbe90600f90 2024-12-09T10:41:40,572 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. as already flushing 2024-12-09T10:41:40,594 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:40,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50422 deadline: 1733740960589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:40,595 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:40,595 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:40,595 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:40,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50408 deadline: 1733740960590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:40,595 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:40,595 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:40,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50390 deadline: 1733740960590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:40,595 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:40,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50340 deadline: 1733740960591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:40,596 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:40,596 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:40,596 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:40,595 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 7ea43b39fc24,36101,1733740827678 for 'test_row_2' in a9bc2b5e1505540f70456fbe90600f90 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T10:41:40,596 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:40,596 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:40,596 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:40,596 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:40,596 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:40,596 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:40,598 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:40,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50410 deadline: 1733740960594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:40,598 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:40,598 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:40,598 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:40,661 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=68 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/B/0d23e59a4089427e9895a130af9c4713 2024-12-09T10:41:40,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/C/9d7dd64163954372b82a148680d83fca is 50, key is test_row_0/C:col10/1733740898421/Put/seqid=0 2024-12-09T10:41:40,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742078_1254 (size=12001) 2024-12-09T10:41:40,702 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:40,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50422 deadline: 1733740960700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:40,703 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:40,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50390 deadline: 1733740960700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:40,703 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:40,703 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:40,703 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:40,703 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:40,703 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:40,703 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:40,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50340 deadline: 1733740960700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:40,703 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:40,704 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:40,704 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:40,704 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:40,713 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:40,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50410 deadline: 1733740960709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:40,713 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:40,713 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:40,713 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:40,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-09T10:41:40,913 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:40,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50422 deadline: 1733740960909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:40,913 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:40,913 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:40,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50390 deadline: 1733740960909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:40,913 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:40,913 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:40,914 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:40,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50340 deadline: 1733740960909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:40,914 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:40,914 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:40,914 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:40,914 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:40,914 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:40,914 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:40,923 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:40,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50410 deadline: 1733740960919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:40,923 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:40,923 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:40,923 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:41,088 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=68 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/C/9d7dd64163954372b82a148680d83fca 2024-12-09T10:41:41,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/A/b8e20005785443c58e347a18f9f8d8bf as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/b8e20005785443c58e347a18f9f8d8bf 2024-12-09T10:41:41,097 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/b8e20005785443c58e347a18f9f8d8bf, entries=150, sequenceid=68, filesize=11.7 K 2024-12-09T10:41:41,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/B/0d23e59a4089427e9895a130af9c4713 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/0d23e59a4089427e9895a130af9c4713 2024-12-09T10:41:41,101 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/0d23e59a4089427e9895a130af9c4713, entries=150, sequenceid=68, filesize=11.7 K 2024-12-09T10:41:41,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/C/9d7dd64163954372b82a148680d83fca as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/9d7dd64163954372b82a148680d83fca 2024-12-09T10:41:41,106 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/9d7dd64163954372b82a148680d83fca, entries=150, sequenceid=68, filesize=11.7 K 2024-12-09T10:41:41,107 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for a9bc2b5e1505540f70456fbe90600f90 in 1313ms, sequenceid=68, compaction requested=false 2024-12-09T10:41:41,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2603): Flush status journal for a9bc2b5e1505540f70456fbe90600f90: 2024-12-09T10:41:41,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:41,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-12-09T10:41:41,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-12-09T10:41:41,110 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-12-09T10:41:41,110 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3860 sec 2024-12-09T10:41:41,112 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees in 2.3910 sec 2024-12-09T10:41:41,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(8855): Flush requested on a9bc2b5e1505540f70456fbe90600f90 2024-12-09T10:41:41,222 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a9bc2b5e1505540f70456fbe90600f90 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-09T10:41:41,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9bc2b5e1505540f70456fbe90600f90, store=A 2024-12-09T10:41:41,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:41:41,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9bc2b5e1505540f70456fbe90600f90, store=B 2024-12-09T10:41:41,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:41:41,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9bc2b5e1505540f70456fbe90600f90, store=C 2024-12-09T10:41:41,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:41:41,228 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/A/091e62da52bc4eec8dfea0254396a1a0 is 50, key is test_row_0/A:col10/1733740901222/Put/seqid=0 2024-12-09T10:41:41,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742079_1255 (size=12001) 2024-12-09T10:41:41,239 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/A/091e62da52bc4eec8dfea0254396a1a0 2024-12-09T10:41:41,245 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/B/410ea7949ac64800afefaed9508f5acb is 50, key is test_row_0/B:col10/1733740901222/Put/seqid=0 2024-12-09T10:41:41,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742080_1256 (size=12001) 2024-12-09T10:41:41,272 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:41,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50340 deadline: 1733740961267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:41,273 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:41,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:41,273 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:41,273 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:41,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50410 deadline: 1733740961268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:41,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:41,273 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:41,273 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:41,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50390 deadline: 1733740961268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:41,274 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:41,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:41,274 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:41,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:41,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:41,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50422 deadline: 1733740961269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:41,274 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:41,274 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:41,274 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:41,381 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:41,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50340 deadline: 1733740961379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:41,382 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:41,382 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:41,382 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:41,382 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:41,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50410 deadline: 1733740961379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:41,383 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:41,383 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:41,383 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:41,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50390 deadline: 1733740961379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:41,383 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:41,383 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:41,383 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:41,383 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:41,383 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:41,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50422 deadline: 1733740961379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:41,384 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:41,384 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:41,384 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:41,593 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:41,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50340 deadline: 1733740961589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:41,593 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:41,593 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:41,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:41,593 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:41,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50410 deadline: 1733740961589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:41,593 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:41,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50390 deadline: 1733740961589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:41,594 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:41,594 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:41,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50422 deadline: 1733740961589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:41,594 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:41,594 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:41,594 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:41,594 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:41,594 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:41,594 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:41,594 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:41,594 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:41,651 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/B/410ea7949ac64800afefaed9508f5acb 2024-12-09T10:41:41,659 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/C/79c84d61d2e94d94ad643bdea8de05ce is 50, key is test_row_0/C:col10/1733740901222/Put/seqid=0 2024-12-09T10:41:41,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742081_1257 (size=12001) 2024-12-09T10:41:41,902 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:41,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50340 deadline: 1733740961899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:41,903 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:41,903 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:41,903 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:41,903 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:41,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50410 deadline: 1733740961899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:41,903 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:41,903 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:41,903 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:41,903 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:41,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50390 deadline: 1733740961899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:41,903 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:41,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50422 deadline: 1733740961899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:41,904 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:41,904 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:41,904 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:41,904 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:41,904 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:41,904 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:42,068 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/C/79c84d61d2e94d94ad643bdea8de05ce 2024-12-09T10:41:42,073 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/A/091e62da52bc4eec8dfea0254396a1a0 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/091e62da52bc4eec8dfea0254396a1a0 2024-12-09T10:41:42,077 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/091e62da52bc4eec8dfea0254396a1a0, entries=150, sequenceid=86, filesize=11.7 K 2024-12-09T10:41:42,078 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/B/410ea7949ac64800afefaed9508f5acb as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/410ea7949ac64800afefaed9508f5acb 2024-12-09T10:41:42,082 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/410ea7949ac64800afefaed9508f5acb, entries=150, sequenceid=86, filesize=11.7 K 2024-12-09T10:41:42,083 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/C/79c84d61d2e94d94ad643bdea8de05ce as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/79c84d61d2e94d94ad643bdea8de05ce 2024-12-09T10:41:42,087 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/79c84d61d2e94d94ad643bdea8de05ce, entries=150, sequenceid=86, filesize=11.7 K 2024-12-09T10:41:42,088 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=87.22 KB/89310 for a9bc2b5e1505540f70456fbe90600f90 in 866ms, sequenceid=86, compaction requested=true 2024-12-09T10:41:42,088 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a9bc2b5e1505540f70456fbe90600f90: 2024-12-09T10:41:42,088 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9bc2b5e1505540f70456fbe90600f90:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T10:41:42,088 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:41:42,088 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T10:41:42,088 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9bc2b5e1505540f70456fbe90600f90:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T10:41:42,088 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:41:42,088 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9bc2b5e1505540f70456fbe90600f90:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T10:41:42,088 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T10:41:42,088 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T10:41:42,089 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T10:41:42,089 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1541): a9bc2b5e1505540f70456fbe90600f90/A is initiating minor compaction (all files) 2024-12-09T10:41:42,090 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a9bc2b5e1505540f70456fbe90600f90/A in TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:42,090 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/41528c70c5c4471084f0037919b20b35, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/b8e20005785443c58e347a18f9f8d8bf, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/091e62da52bc4eec8dfea0254396a1a0] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp, totalSize=35.3 K 2024-12-09T10:41:42,090 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T10:41:42,090 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1541): a9bc2b5e1505540f70456fbe90600f90/B is initiating minor compaction (all files) 2024-12-09T10:41:42,090 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a9bc2b5e1505540f70456fbe90600f90/B in TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:42,090 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/7ffe5f63c06846bea7fb035e9a61eaf7, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/0d23e59a4089427e9895a130af9c4713, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/410ea7949ac64800afefaed9508f5acb] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp, totalSize=35.3 K 2024-12-09T10:41:42,090 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 41528c70c5c4471084f0037919b20b35, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733740898401 2024-12-09T10:41:42,090 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ffe5f63c06846bea7fb035e9a61eaf7, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733740898401 2024-12-09T10:41:42,091 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting b8e20005785443c58e347a18f9f8d8bf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1733740898421 2024-12-09T10:41:42,091 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d23e59a4089427e9895a130af9c4713, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1733740898421 2024-12-09T10:41:42,091 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 091e62da52bc4eec8dfea0254396a1a0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1733740900592 2024-12-09T10:41:42,092 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 410ea7949ac64800afefaed9508f5acb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1733740900592 2024-12-09T10:41:42,099 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a9bc2b5e1505540f70456fbe90600f90#A#compaction#213 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:41:42,099 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/A/c23fa272d24c4008abced08ff5154d9c is 50, key is test_row_0/A:col10/1733740901222/Put/seqid=0 2024-12-09T10:41:42,106 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a9bc2b5e1505540f70456fbe90600f90#B#compaction#214 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:41:42,107 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/B/59efd1b2f91240358a14c3756581b2a3 is 50, key is test_row_0/B:col10/1733740901222/Put/seqid=0 2024-12-09T10:41:42,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742082_1258 (size=12207) 2024-12-09T10:41:42,113 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/A/c23fa272d24c4008abced08ff5154d9c as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/c23fa272d24c4008abced08ff5154d9c 2024-12-09T10:41:42,118 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a9bc2b5e1505540f70456fbe90600f90/A of a9bc2b5e1505540f70456fbe90600f90 into c23fa272d24c4008abced08ff5154d9c(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:41:42,118 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a9bc2b5e1505540f70456fbe90600f90: 2024-12-09T10:41:42,118 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., storeName=a9bc2b5e1505540f70456fbe90600f90/A, priority=13, startTime=1733740902088; duration=0sec 2024-12-09T10:41:42,119 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T10:41:42,119 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9bc2b5e1505540f70456fbe90600f90:A 2024-12-09T10:41:42,119 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T10:41:42,120 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T10:41:42,120 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1541): a9bc2b5e1505540f70456fbe90600f90/C is initiating minor compaction (all files) 2024-12-09T10:41:42,120 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a9bc2b5e1505540f70456fbe90600f90/C in TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:42,120 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/9d2489362d24431ba9bd18c40d9b2f8d, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/9d7dd64163954372b82a148680d83fca, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/79c84d61d2e94d94ad643bdea8de05ce] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp, totalSize=35.3 K 2024-12-09T10:41:42,120 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9d2489362d24431ba9bd18c40d9b2f8d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733740898401 2024-12-09T10:41:42,121 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9d7dd64163954372b82a148680d83fca, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1733740898421 2024-12-09T10:41:42,121 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 79c84d61d2e94d94ad643bdea8de05ce, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1733740900592 2024-12-09T10:41:42,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742083_1259 (size=12207) 2024-12-09T10:41:42,129 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/B/59efd1b2f91240358a14c3756581b2a3 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/59efd1b2f91240358a14c3756581b2a3 2024-12-09T10:41:42,132 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a9bc2b5e1505540f70456fbe90600f90#C#compaction#215 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:41:42,133 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/C/c36e92c81e094bdfa7588cf23c25a8ba is 50, key is test_row_0/C:col10/1733740901222/Put/seqid=0 2024-12-09T10:41:42,136 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a9bc2b5e1505540f70456fbe90600f90/B of a9bc2b5e1505540f70456fbe90600f90 into 59efd1b2f91240358a14c3756581b2a3(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:41:42,136 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a9bc2b5e1505540f70456fbe90600f90: 2024-12-09T10:41:42,136 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., storeName=a9bc2b5e1505540f70456fbe90600f90/B, priority=13, startTime=1733740902088; duration=0sec 2024-12-09T10:41:42,136 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:41:42,136 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9bc2b5e1505540f70456fbe90600f90:B 2024-12-09T10:41:42,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742084_1260 (size=12207) 2024-12-09T10:41:42,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(8855): Flush requested on a9bc2b5e1505540f70456fbe90600f90 2024-12-09T10:41:42,414 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a9bc2b5e1505540f70456fbe90600f90 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-09T10:41:42,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9bc2b5e1505540f70456fbe90600f90, store=A 2024-12-09T10:41:42,415 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:41:42,415 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9bc2b5e1505540f70456fbe90600f90, store=B 2024-12-09T10:41:42,415 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:41:42,415 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9bc2b5e1505540f70456fbe90600f90, store=C 2024-12-09T10:41:42,415 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:41:42,420 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/A/5bd267c6b30e49688ce75cb65465303f is 50, key is test_row_0/A:col10/1733740901265/Put/seqid=0 2024-12-09T10:41:42,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742085_1261 (size=12001) 2024-12-09T10:41:42,426 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=105 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/A/5bd267c6b30e49688ce75cb65465303f 2024-12-09T10:41:42,439 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/B/60c47024b4174321ad27b2a7623e8e19 is 50, key is test_row_0/B:col10/1733740901265/Put/seqid=0 2024-12-09T10:41:42,440 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:42,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50390 deadline: 1733740962434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:42,441 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:42,441 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:42,441 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:42,441 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:42,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50410 deadline: 1733740962435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:42,442 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:42,442 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:42,442 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:42,442 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:42,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50340 deadline: 1733740962437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:42,442 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:42,442 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:42,442 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:42,442 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:42,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50422 deadline: 1733740962437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:42,443 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:42,443 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:42,443 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:42,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742086_1262 (size=12001) 2024-12-09T10:41:42,446 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=105 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/B/60c47024b4174321ad27b2a7623e8e19 2024-12-09T10:41:42,458 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/C/10b4d8ff98f449b89b3eebdaf9f62c69 is 50, key is test_row_0/C:col10/1733740901265/Put/seqid=0 2024-12-09T10:41:42,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742087_1263 (size=12001) 2024-12-09T10:41:42,552 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/C/c36e92c81e094bdfa7588cf23c25a8ba as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/c36e92c81e094bdfa7588cf23c25a8ba 2024-12-09T10:41:42,553 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:42,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50410 deadline: 1733740962549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:42,554 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:42,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50390 deadline: 1733740962549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:42,554 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:42,554 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:42,554 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:42,554 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:42,554 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:42,554 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:42,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50422 deadline: 1733740962550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:42,554 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:42,554 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:42,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50340 deadline: 1733740962550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:42,555 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:42,555 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:42,555 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:42,562 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:42,562 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:42,562 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:42,563 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a9bc2b5e1505540f70456fbe90600f90/C of a9bc2b5e1505540f70456fbe90600f90 into c36e92c81e094bdfa7588cf23c25a8ba(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:41:42,563 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a9bc2b5e1505540f70456fbe90600f90: 2024-12-09T10:41:42,563 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., storeName=a9bc2b5e1505540f70456fbe90600f90/C, priority=13, startTime=1733740902088; duration=0sec 2024-12-09T10:41:42,563 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:41:42,563 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9bc2b5e1505540f70456fbe90600f90:C 2024-12-09T10:41:42,612 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:42,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50408 deadline: 1733740962609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:42,613 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 7ea43b39fc24,36101,1733740827678 for 'test_row_2' in a9bc2b5e1505540f70456fbe90600f90 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T10:41:42,614 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:42,614 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:42,614 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:42,761 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:42,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50390 deadline: 1733740962760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:42,761 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:42,761 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:42,761 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:42,762 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:42,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50410 deadline: 1733740962760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:42,762 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:42,762 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:42,762 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:42,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50340 deadline: 1733740962760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:42,762 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:42,762 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:42,762 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:42,762 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:42,772 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:42,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50422 deadline: 1733740962769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:42,772 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:42,773 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:42,773 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:42,876 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=105 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/C/10b4d8ff98f449b89b3eebdaf9f62c69 2024-12-09T10:41:42,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-09T10:41:42,880 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T10:41:42,880 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-09T10:41:42,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-12-09T10:41:42,882 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/A/5bd267c6b30e49688ce75cb65465303f as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/5bd267c6b30e49688ce75cb65465303f 2024-12-09T10:41:42,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-09T10:41:42,882 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T10:41:42,883 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T10:41:42,883 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T10:41:42,886 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/5bd267c6b30e49688ce75cb65465303f, entries=150, sequenceid=105, filesize=11.7 K 2024-12-09T10:41:42,887 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/B/60c47024b4174321ad27b2a7623e8e19 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/60c47024b4174321ad27b2a7623e8e19 2024-12-09T10:41:42,892 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/60c47024b4174321ad27b2a7623e8e19, entries=150, sequenceid=105, filesize=11.7 K 2024-12-09T10:41:42,893 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/C/10b4d8ff98f449b89b3eebdaf9f62c69 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/10b4d8ff98f449b89b3eebdaf9f62c69 2024-12-09T10:41:42,897 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/10b4d8ff98f449b89b3eebdaf9f62c69, entries=150, sequenceid=105, filesize=11.7 K 2024-12-09T10:41:42,898 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for a9bc2b5e1505540f70456fbe90600f90 in 484ms, sequenceid=105, compaction requested=false 2024-12-09T10:41:42,898 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a9bc2b5e1505540f70456fbe90600f90: 2024-12-09T10:41:42,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-09T10:41:43,036 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-09T10:41:43,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:43,036 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing a9bc2b5e1505540f70456fbe90600f90 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-09T10:41:43,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9bc2b5e1505540f70456fbe90600f90, store=A 2024-12-09T10:41:43,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:41:43,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9bc2b5e1505540f70456fbe90600f90, store=B 2024-12-09T10:41:43,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:41:43,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9bc2b5e1505540f70456fbe90600f90, store=C 2024-12-09T10:41:43,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:41:43,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/A/401b23b8cc244183a8a21482bb6e12cd is 50, key is test_row_0/A:col10/1733740902435/Put/seqid=0 2024-12-09T10:41:43,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742088_1264 (size=12001) 2024-12-09T10:41:43,047 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/A/401b23b8cc244183a8a21482bb6e12cd 2024-12-09T10:41:43,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/B/e6e8752c2bd64658abd3d9599c6c8931 is 50, key is test_row_0/B:col10/1733740902435/Put/seqid=0 2024-12-09T10:41:43,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742089_1265 (size=12001) 2024-12-09T10:41:43,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(8855): Flush requested on a9bc2b5e1505540f70456fbe90600f90 2024-12-09T10:41:43,072 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. as already flushing 2024-12-09T10:41:43,093 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:43,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50390 deadline: 1733740963089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:43,093 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:43,094 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:43,094 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:43,095 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:43,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50410 deadline: 1733740963090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:43,095 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:43,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50422 deadline: 1733740963090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:43,096 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:43,096 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:43,096 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:43,096 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:43,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50340 deadline: 1733740963091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:43,096 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:43,096 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:43,096 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:43,096 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:43,096 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:43,096 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:43,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-09T10:41:43,204 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:43,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50410 deadline: 1733740963200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:43,204 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:43,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50340 deadline: 1733740963200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:43,204 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:43,204 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:43,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50390 deadline: 1733740963200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:43,204 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:43,204 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:43,204 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:43,204 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:43,204 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:43,204 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:43,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50422 deadline: 1733740963200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:43,205 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:43,205 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:43,205 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:43,205 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:43,205 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:43,205 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:43,412 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:43,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50340 deadline: 1733740963409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:43,413 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:43,412 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:43,413 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:43,413 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:43,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50422 deadline: 1733740963409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:43,413 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:43,413 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:43,413 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:43,413 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:43,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50390 deadline: 1733740963409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:43,413 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:43,413 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:43,413 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:43,413 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:43,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50410 deadline: 1733740963409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:43,414 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:43,414 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:43,414 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:43,460 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/B/e6e8752c2bd64658abd3d9599c6c8931 2024-12-09T10:41:43,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/C/5218bc8f9e87430f9c9d7323b3eff0e9 is 50, key is test_row_0/C:col10/1733740902435/Put/seqid=0 2024-12-09T10:41:43,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742090_1266 (size=12001) 2024-12-09T10:41:43,507 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/C/5218bc8f9e87430f9c9d7323b3eff0e9 2024-12-09T10:41:43,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-09T10:41:43,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/A/401b23b8cc244183a8a21482bb6e12cd as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/401b23b8cc244183a8a21482bb6e12cd 2024-12-09T10:41:43,523 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/401b23b8cc244183a8a21482bb6e12cd, entries=150, sequenceid=121, filesize=11.7 K 2024-12-09T10:41:43,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/B/e6e8752c2bd64658abd3d9599c6c8931 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/e6e8752c2bd64658abd3d9599c6c8931 2024-12-09T10:41:43,532 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/e6e8752c2bd64658abd3d9599c6c8931, entries=150, sequenceid=121, filesize=11.7 K 2024-12-09T10:41:43,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/C/5218bc8f9e87430f9c9d7323b3eff0e9 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/5218bc8f9e87430f9c9d7323b3eff0e9 2024-12-09T10:41:43,538 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/5218bc8f9e87430f9c9d7323b3eff0e9, entries=150, sequenceid=121, filesize=11.7 K 2024-12-09T10:41:43,539 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for a9bc2b5e1505540f70456fbe90600f90 in 503ms, sequenceid=121, compaction requested=true 2024-12-09T10:41:43,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for a9bc2b5e1505540f70456fbe90600f90: 2024-12-09T10:41:43,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:43,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-12-09T10:41:43,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-12-09T10:41:43,543 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-12-09T10:41:43,543 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 658 msec 2024-12-09T10:41:43,546 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 664 msec 2024-12-09T10:41:43,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(8855): Flush requested on a9bc2b5e1505540f70456fbe90600f90 2024-12-09T10:41:43,722 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a9bc2b5e1505540f70456fbe90600f90 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-09T10:41:43,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9bc2b5e1505540f70456fbe90600f90, store=A 2024-12-09T10:41:43,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:41:43,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9bc2b5e1505540f70456fbe90600f90, store=B 2024-12-09T10:41:43,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:41:43,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9bc2b5e1505540f70456fbe90600f90, store=C 2024-12-09T10:41:43,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:41:43,732 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/A/298a0d7f7bbb4c44a87a2700bb845f41 is 50, key is test_row_0/A:col10/1733740903090/Put/seqid=0 2024-12-09T10:41:43,742 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:43,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50390 deadline: 1733740963734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:43,743 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:43,743 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:43,743 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:43,747 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:43,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50340 deadline: 1733740963741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:43,748 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:43,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50422 deadline: 1733740963742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:43,748 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:43,748 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:43,748 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:43,748 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:43,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50410 deadline: 1733740963743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:43,748 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:43,748 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:43,749 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:43,749 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:43,749 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:43,749 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:43,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742091_1267 (size=12151) 2024-12-09T10:41:43,852 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:43,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50390 deadline: 1733740963849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:43,853 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:43,853 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:43,853 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:43,862 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:43,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50340 deadline: 1733740963860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:43,862 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:43,862 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:43,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50422 deadline: 1733740963860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:43,863 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:43,863 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:43,863 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:43,863 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:43,863 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:43,864 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:43,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50410 deadline: 1733740963861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:43,865 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:43,865 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:43,865 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:44,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-09T10:41:44,020 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T10:41:44,021 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-09T10:41:44,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-12-09T10:41:44,023 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T10:41:44,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-09T10:41:44,023 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T10:41:44,024 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T10:41:44,063 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:44,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50390 deadline: 1733740964060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:44,064 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:44,064 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:44,064 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:44,072 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:44,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50340 deadline: 1733740964070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:44,073 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:44,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50422 deadline: 1733740964070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:44,073 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:44,073 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:44,073 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:44,073 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:44,073 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:44,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50410 deadline: 1733740964070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:44,073 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:44,074 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:44,074 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:44,074 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:44,074 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:44,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-09T10:41:44,154 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/A/298a0d7f7bbb4c44a87a2700bb845f41 2024-12-09T10:41:44,168 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/B/ae1722c9079a43f6bb386c6d779398ab is 50, key is test_row_0/B:col10/1733740903090/Put/seqid=0 2024-12-09T10:41:44,177 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-09T10:41:44,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:44,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. as already flushing 2024-12-09T10:41:44,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:44,177 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:44,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:44,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:44,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742092_1268 (size=12151) 2024-12-09T10:41:44,331 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-09T10:41:44,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:44,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. as already flushing 2024-12-09T10:41:44,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:44,331 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:44,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:44,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:44,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-09T10:41:44,373 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:44,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50390 deadline: 1733740964370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:44,374 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:44,374 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:44,374 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:44,382 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:44,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50340 deadline: 1733740964380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:44,382 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:44,382 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:44,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50410 deadline: 1733740964380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:44,382 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:44,382 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:44,383 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:44,383 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:44,383 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:44,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50422 deadline: 1733740964380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:44,383 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:44,383 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:44,383 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:44,383 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:44,485 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-09T10:41:44,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:44,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. as already flushing 2024-12-09T10:41:44,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:44,485 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:44,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:44,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:44,604 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/B/ae1722c9079a43f6bb386c6d779398ab 2024-12-09T10:41:44,616 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/C/141e2502e0584f848ac0003925dc83b8 is 50, key is test_row_0/C:col10/1733740903090/Put/seqid=0 2024-12-09T10:41:44,639 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-09T10:41:44,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:44,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. as already flushing 2024-12-09T10:41:44,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:44,640 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:44,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:44,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:44,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-09T10:41:44,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742093_1269 (size=12151) 2024-12-09T10:41:44,794 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-09T10:41:44,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:44,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. as already flushing 2024-12-09T10:41:44,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:44,794 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:44,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:44,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:44,885 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:44,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50390 deadline: 1733740964880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:44,886 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:44,886 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:44,886 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:44,899 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:44,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50340 deadline: 1733740964890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:44,900 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:44,900 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:44,900 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:44,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50410 deadline: 1733740964890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:44,900 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:44,900 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:44,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50422 deadline: 1733740964890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:44,900 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:44,900 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:44,900 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:44,900 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:44,900 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:44,900 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:44,948 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-09T10:41:44,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:44,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. as already flushing 2024-12-09T10:41:44,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:44,949 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:44,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:44,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:45,070 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/C/141e2502e0584f848ac0003925dc83b8 2024-12-09T10:41:45,076 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/A/298a0d7f7bbb4c44a87a2700bb845f41 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/298a0d7f7bbb4c44a87a2700bb845f41 2024-12-09T10:41:45,081 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/298a0d7f7bbb4c44a87a2700bb845f41, entries=150, sequenceid=140, filesize=11.9 K 2024-12-09T10:41:45,083 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/B/ae1722c9079a43f6bb386c6d779398ab as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/ae1722c9079a43f6bb386c6d779398ab 2024-12-09T10:41:45,088 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/ae1722c9079a43f6bb386c6d779398ab, entries=150, sequenceid=140, filesize=11.9 K 2024-12-09T10:41:45,089 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/C/141e2502e0584f848ac0003925dc83b8 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/141e2502e0584f848ac0003925dc83b8 2024-12-09T10:41:45,092 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/141e2502e0584f848ac0003925dc83b8, entries=150, sequenceid=140, filesize=11.9 K 2024-12-09T10:41:45,093 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for a9bc2b5e1505540f70456fbe90600f90 in 1371ms, sequenceid=140, compaction requested=true 2024-12-09T10:41:45,093 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a9bc2b5e1505540f70456fbe90600f90: 2024-12-09T10:41:45,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9bc2b5e1505540f70456fbe90600f90:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T10:41:45,093 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T10:41:45,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:41:45,094 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T10:41:45,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9bc2b5e1505540f70456fbe90600f90:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T10:41:45,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:41:45,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9bc2b5e1505540f70456fbe90600f90:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T10:41:45,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T10:41:45,096 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48360 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T10:41:45,096 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1541): a9bc2b5e1505540f70456fbe90600f90/A is initiating minor compaction (all files) 2024-12-09T10:41:45,096 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a9bc2b5e1505540f70456fbe90600f90/A in TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:45,096 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/c23fa272d24c4008abced08ff5154d9c, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/5bd267c6b30e49688ce75cb65465303f, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/401b23b8cc244183a8a21482bb6e12cd, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/298a0d7f7bbb4c44a87a2700bb845f41] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp, totalSize=47.2 K 2024-12-09T10:41:45,097 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48360 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T10:41:45,097 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1541): a9bc2b5e1505540f70456fbe90600f90/B is initiating minor compaction (all files) 2024-12-09T10:41:45,097 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting c23fa272d24c4008abced08ff5154d9c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1733740900592 2024-12-09T10:41:45,098 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a9bc2b5e1505540f70456fbe90600f90/B in TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:45,098 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/59efd1b2f91240358a14c3756581b2a3, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/60c47024b4174321ad27b2a7623e8e19, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/e6e8752c2bd64658abd3d9599c6c8931, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/ae1722c9079a43f6bb386c6d779398ab] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp, totalSize=47.2 K 2024-12-09T10:41:45,098 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5bd267c6b30e49688ce75cb65465303f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1733740901265 2024-12-09T10:41:45,098 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 59efd1b2f91240358a14c3756581b2a3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1733740900592 2024-12-09T10:41:45,098 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 401b23b8cc244183a8a21482bb6e12cd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1733740902432 2024-12-09T10:41:45,098 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 60c47024b4174321ad27b2a7623e8e19, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1733740901265 2024-12-09T10:41:45,099 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 298a0d7f7bbb4c44a87a2700bb845f41, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1733740903089 2024-12-09T10:41:45,099 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting e6e8752c2bd64658abd3d9599c6c8931, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1733740902432 2024-12-09T10:41:45,099 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting ae1722c9079a43f6bb386c6d779398ab, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1733740903089 2024-12-09T10:41:45,102 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-09T10:41:45,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:45,103 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing a9bc2b5e1505540f70456fbe90600f90 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-09T10:41:45,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9bc2b5e1505540f70456fbe90600f90, store=A 2024-12-09T10:41:45,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:41:45,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9bc2b5e1505540f70456fbe90600f90, store=B 2024-12-09T10:41:45,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:41:45,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9bc2b5e1505540f70456fbe90600f90, store=C 2024-12-09T10:41:45,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:41:45,111 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a9bc2b5e1505540f70456fbe90600f90#B#compaction#225 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:41:45,112 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/B/515e28dc83924127bb5e582d354544ef is 50, key is test_row_0/B:col10/1733740903090/Put/seqid=0 2024-12-09T10:41:45,113 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a9bc2b5e1505540f70456fbe90600f90#A#compaction#226 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:41:45,113 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/A/4ee57521787443d789dce1c957e97870 is 50, key is test_row_0/A:col10/1733740903090/Put/seqid=0 2024-12-09T10:41:45,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/A/276a443b02574b11a7bc2d1c7cf15c66 is 50, key is test_row_0/A:col10/1733740903732/Put/seqid=0 2024-12-09T10:41:45,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742095_1271 (size=12493) 2024-12-09T10:41:45,151 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/A/4ee57521787443d789dce1c957e97870 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/4ee57521787443d789dce1c957e97870 2024-12-09T10:41:45,157 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a9bc2b5e1505540f70456fbe90600f90/A of a9bc2b5e1505540f70456fbe90600f90 into 4ee57521787443d789dce1c957e97870(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:41:45,157 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a9bc2b5e1505540f70456fbe90600f90: 2024-12-09T10:41:45,157 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., storeName=a9bc2b5e1505540f70456fbe90600f90/A, priority=12, startTime=1733740905093; duration=0sec 2024-12-09T10:41:45,158 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T10:41:45,158 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9bc2b5e1505540f70456fbe90600f90:A 2024-12-09T10:41:45,158 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T10:41:45,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-09T10:41:45,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742096_1272 (size=12151) 2024-12-09T10:41:45,162 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48360 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T10:41:45,162 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1541): a9bc2b5e1505540f70456fbe90600f90/C is initiating minor compaction (all files) 2024-12-09T10:41:45,163 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a9bc2b5e1505540f70456fbe90600f90/C in TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:45,163 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/c36e92c81e094bdfa7588cf23c25a8ba, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/10b4d8ff98f449b89b3eebdaf9f62c69, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/5218bc8f9e87430f9c9d7323b3eff0e9, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/141e2502e0584f848ac0003925dc83b8] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp, totalSize=47.2 K 2024-12-09T10:41:45,163 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting c36e92c81e094bdfa7588cf23c25a8ba, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1733740900592 2024-12-09T10:41:45,164 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 10b4d8ff98f449b89b3eebdaf9f62c69, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1733740901265 2024-12-09T10:41:45,164 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5218bc8f9e87430f9c9d7323b3eff0e9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1733740902432 2024-12-09T10:41:45,164 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 141e2502e0584f848ac0003925dc83b8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1733740903089 2024-12-09T10:41:45,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742094_1270 (size=12493) 2024-12-09T10:41:45,191 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a9bc2b5e1505540f70456fbe90600f90#C#compaction#228 average throughput is 0.50 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:41:45,193 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/C/a727e020dd764dc09c393485c0908644 is 50, key is test_row_0/C:col10/1733740903090/Put/seqid=0 2024-12-09T10:41:45,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742097_1273 (size=12493) 2024-12-09T10:41:45,208 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/C/a727e020dd764dc09c393485c0908644 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/a727e020dd764dc09c393485c0908644 2024-12-09T10:41:45,215 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a9bc2b5e1505540f70456fbe90600f90/C of a9bc2b5e1505540f70456fbe90600f90 into a727e020dd764dc09c393485c0908644(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:41:45,215 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a9bc2b5e1505540f70456fbe90600f90: 2024-12-09T10:41:45,215 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., storeName=a9bc2b5e1505540f70456fbe90600f90/C, priority=12, startTime=1733740905094; duration=0sec 2024-12-09T10:41:45,215 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:41:45,215 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9bc2b5e1505540f70456fbe90600f90:C 2024-12-09T10:41:45,563 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/A/276a443b02574b11a7bc2d1c7cf15c66 2024-12-09T10:41:45,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/B/5118f60a7f854379a04dd8f38ba1044a is 50, key is test_row_0/B:col10/1733740903732/Put/seqid=0 2024-12-09T10:41:45,579 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/B/515e28dc83924127bb5e582d354544ef as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/515e28dc83924127bb5e582d354544ef 2024-12-09T10:41:45,592 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a9bc2b5e1505540f70456fbe90600f90/B of a9bc2b5e1505540f70456fbe90600f90 into 515e28dc83924127bb5e582d354544ef(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:41:45,592 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a9bc2b5e1505540f70456fbe90600f90: 2024-12-09T10:41:45,592 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., storeName=a9bc2b5e1505540f70456fbe90600f90/B, priority=12, startTime=1733740905094; duration=0sec 2024-12-09T10:41:45,593 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:41:45,593 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9bc2b5e1505540f70456fbe90600f90:B 2024-12-09T10:41:45,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742098_1274 (size=12151) 2024-12-09T10:41:45,625 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/B/5118f60a7f854379a04dd8f38ba1044a 2024-12-09T10:41:45,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/C/9e88f495ee964ef4acf875676dfc1818 is 50, key is test_row_0/C:col10/1733740903732/Put/seqid=0 2024-12-09T10:41:45,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742099_1275 (size=12151) 2024-12-09T10:41:45,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(8855): Flush requested on a9bc2b5e1505540f70456fbe90600f90 2024-12-09T10:41:45,902 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. as already flushing 2024-12-09T10:41:45,937 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:45,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50410 deadline: 1733740965930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:45,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:45,938 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:45,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:45,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:45,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50390 deadline: 1733740965931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:45,938 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:45,938 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:45,938 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:45,942 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:45,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50340 deadline: 1733740965937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:45,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:45,943 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:45,943 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:45,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50422 deadline: 1733740965937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:45,943 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:45,944 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:45,944 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:45,944 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:46,052 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:46,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50410 deadline: 1733740966049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:46,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:46,053 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:46,053 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:46,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50390 deadline: 1733740966050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:46,053 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:46,053 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:46,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:46,053 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:46,053 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:46,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50340 deadline: 1733740966050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:46,054 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:46,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:46,054 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:46,054 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:46,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50422 deadline: 1733740966050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:46,054 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:46,054 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:46,054 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:46,075 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/C/9e88f495ee964ef4acf875676dfc1818 2024-12-09T10:41:46,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/A/276a443b02574b11a7bc2d1c7cf15c66 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/276a443b02574b11a7bc2d1c7cf15c66 2024-12-09T10:41:46,086 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/276a443b02574b11a7bc2d1c7cf15c66, entries=150, sequenceid=153, filesize=11.9 K 2024-12-09T10:41:46,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/B/5118f60a7f854379a04dd8f38ba1044a as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/5118f60a7f854379a04dd8f38ba1044a 2024-12-09T10:41:46,092 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/5118f60a7f854379a04dd8f38ba1044a, entries=150, sequenceid=153, filesize=11.9 K 2024-12-09T10:41:46,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/C/9e88f495ee964ef4acf875676dfc1818 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/9e88f495ee964ef4acf875676dfc1818 2024-12-09T10:41:46,098 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/9e88f495ee964ef4acf875676dfc1818, entries=150, sequenceid=153, filesize=11.9 K 2024-12-09T10:41:46,099 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for a9bc2b5e1505540f70456fbe90600f90 in 996ms, sequenceid=153, compaction requested=false 2024-12-09T10:41:46,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for a9bc2b5e1505540f70456fbe90600f90: 2024-12-09T10:41:46,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:46,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-12-09T10:41:46,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-12-09T10:41:46,102 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-12-09T10:41:46,102 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0770 sec 2024-12-09T10:41:46,108 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 2.0820 sec 2024-12-09T10:41:46,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-09T10:41:46,180 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T10:41:46,180 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-09T10:41:46,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-12-09T10:41:46,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-09T10:41:46,183 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T10:41:46,184 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T10:41:46,184 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T10:41:46,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(8855): Flush requested on a9bc2b5e1505540f70456fbe90600f90 2024-12-09T10:41:46,264 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a9bc2b5e1505540f70456fbe90600f90 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-09T10:41:46,264 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9bc2b5e1505540f70456fbe90600f90, store=A 2024-12-09T10:41:46,265 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T10:41:46,265 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9bc2b5e1505540f70456fbe90600f90, store=B 2024-12-09T10:41:46,265 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T10:41:46,265 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9bc2b5e1505540f70456fbe90600f90, store=C 2024-12-09T10:41:46,265 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T10:41:46,270 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/A/77d139004e344877b1d3acabeb82cf9a is 50, key is test_row_0/A:col10/1733740905929/Put/seqid=0 2024-12-09T10:41:46,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742100_1276 (size=12151) 2024-12-09T10:41:46,277 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:46,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50390 deadline: 1733740966272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:46,278 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:46,278 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:46,278 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:46,278 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:46,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50422 deadline: 1733740966273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:46,279 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:46,279 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:46,279 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:46,279 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:46,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50340 deadline: 1733740966273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:46,279 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:46,279 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:46,279 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:46,281 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:46,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50410 deadline: 1733740966278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:46,282 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:46,282 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:46,282 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:46,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-09T10:41:46,337 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-09T10:41:46,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:46,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. as already flushing 2024-12-09T10:41:46,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:46,337 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:46,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:46,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:46,393 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:46,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50422 deadline: 1733740966390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:46,394 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:46,394 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:46,394 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:46,394 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:46,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50390 deadline: 1733740966390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:46,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:46,395 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:46,395 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:46,395 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:46,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50340 deadline: 1733740966390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:46,397 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:46,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50410 deadline: 1733740966390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:46,397 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:46,397 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:46,397 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:46,399 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:46,399 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:46,399 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:46,492 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-09T10:41:46,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:46,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. as already flushing 2024-12-09T10:41:46,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:46,493 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:46,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:46,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:46,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-09T10:41:46,602 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:46,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50422 deadline: 1733740966599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:46,602 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:46,602 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:46,603 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:46,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50390 deadline: 1733740966600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:46,603 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:46,603 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:46,603 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:46,603 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:46,603 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:46,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50410 deadline: 1733740966600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:46,604 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:46,604 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:46,604 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:46,612 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:46,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50340 deadline: 1733740966610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:46,613 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:46,613 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:46,613 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:46,623 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:46,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50408 deadline: 1733740966620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:46,624 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 7ea43b39fc24,36101,1733740827678 for 'test_row_2' in a9bc2b5e1505540f70456fbe90600f90 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T10:41:46,624 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:46,624 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:46,624 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:46,645 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-09T10:41:46,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:46,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. as already flushing 2024-12-09T10:41:46,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:46,646 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:46,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:46,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:46,676 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=178 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/A/77d139004e344877b1d3acabeb82cf9a 2024-12-09T10:41:46,685 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/B/261a9256666244368cee662d2c8ba81b is 50, key is test_row_0/B:col10/1733740905929/Put/seqid=0 2024-12-09T10:41:46,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742101_1277 (size=12151) 2024-12-09T10:41:46,800 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-09T10:41:46,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:46,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. as already flushing 2024-12-09T10:41:46,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:46,801 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:46,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:46,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:46,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-09T10:41:46,914 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:46,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50422 deadline: 1733740966910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:46,915 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:46,915 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:46,915 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:46,915 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:46,915 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:46,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50390 deadline: 1733740966910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:46,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50410 deadline: 1733740966910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:46,915 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:46,915 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:46,915 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:46,915 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:46,915 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:46,916 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:46,921 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:46,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50340 deadline: 1733740966920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:46,922 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:46,922 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:46,922 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:46,954 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-09T10:41:46,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:46,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. as already flushing 2024-12-09T10:41:46,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:46,954 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:46,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:46,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:47,089 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=178 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/B/261a9256666244368cee662d2c8ba81b 2024-12-09T10:41:47,097 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/C/182b81033c434657ae315668105bfe40 is 50, key is test_row_0/C:col10/1733740905929/Put/seqid=0 2024-12-09T10:41:47,107 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-09T10:41:47,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:47,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. as already flushing 2024-12-09T10:41:47,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:47,108 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:47,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:47,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:47,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742102_1278 (size=12151) 2024-12-09T10:41:47,261 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-09T10:41:47,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:47,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. as already flushing 2024-12-09T10:41:47,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:47,261 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:47,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:47,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:47,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-09T10:41:47,414 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-09T10:41:47,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:47,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. as already flushing 2024-12-09T10:41:47,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:47,414 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:47,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:47,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:47,423 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:47,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50422 deadline: 1733740967419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:47,424 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:47,424 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:47,424 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:47,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:47,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50340 deadline: 1733740967430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:47,433 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:47,433 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:47,433 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:47,435 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:47,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50410 deadline: 1733740967433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:47,435 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:47,435 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:47,435 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:47,435 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:47,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50390 deadline: 1733740967433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:47,436 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:47,436 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:47,436 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:47,531 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=178 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/C/182b81033c434657ae315668105bfe40 2024-12-09T10:41:47,536 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/A/77d139004e344877b1d3acabeb82cf9a as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/77d139004e344877b1d3acabeb82cf9a 2024-12-09T10:41:47,544 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/77d139004e344877b1d3acabeb82cf9a, entries=150, sequenceid=178, filesize=11.9 K 2024-12-09T10:41:47,545 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/B/261a9256666244368cee662d2c8ba81b as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/261a9256666244368cee662d2c8ba81b 2024-12-09T10:41:47,549 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/261a9256666244368cee662d2c8ba81b, entries=150, sequenceid=178, filesize=11.9 K 2024-12-09T10:41:47,550 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/C/182b81033c434657ae315668105bfe40 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/182b81033c434657ae315668105bfe40 2024-12-09T10:41:47,554 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/182b81033c434657ae315668105bfe40, entries=150, sequenceid=178, filesize=11.9 K 2024-12-09T10:41:47,555 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~379.92 KB/389040, currentSize=46.96 KB/48090 for a9bc2b5e1505540f70456fbe90600f90 in 1292ms, sequenceid=178, compaction requested=true 2024-12-09T10:41:47,555 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a9bc2b5e1505540f70456fbe90600f90: 2024-12-09T10:41:47,555 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9bc2b5e1505540f70456fbe90600f90:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T10:41:47,555 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:41:47,555 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9bc2b5e1505540f70456fbe90600f90:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T10:41:47,555 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:41:47,555 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9bc2b5e1505540f70456fbe90600f90:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T10:41:47,555 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T10:41:47,555 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T10:41:47,555 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T10:41:47,557 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T10:41:47,557 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T10:41:47,557 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1541): a9bc2b5e1505540f70456fbe90600f90/A is initiating minor compaction (all files) 2024-12-09T10:41:47,557 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1541): a9bc2b5e1505540f70456fbe90600f90/B is initiating minor compaction (all files) 2024-12-09T10:41:47,557 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a9bc2b5e1505540f70456fbe90600f90/A in TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:47,557 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a9bc2b5e1505540f70456fbe90600f90/B in TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:47,557 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/515e28dc83924127bb5e582d354544ef, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/5118f60a7f854379a04dd8f38ba1044a, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/261a9256666244368cee662d2c8ba81b] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp, totalSize=35.9 K 2024-12-09T10:41:47,557 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/4ee57521787443d789dce1c957e97870, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/276a443b02574b11a7bc2d1c7cf15c66, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/77d139004e344877b1d3acabeb82cf9a] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp, totalSize=35.9 K 2024-12-09T10:41:47,557 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 515e28dc83924127bb5e582d354544ef, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1733740903089 2024-12-09T10:41:47,557 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ee57521787443d789dce1c957e97870, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1733740903089 2024-12-09T10:41:47,558 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 5118f60a7f854379a04dd8f38ba1044a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1733740903732 2024-12-09T10:41:47,558 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 276a443b02574b11a7bc2d1c7cf15c66, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1733740903732 2024-12-09T10:41:47,559 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 77d139004e344877b1d3acabeb82cf9a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1733740905929 2024-12-09T10:41:47,559 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 261a9256666244368cee662d2c8ba81b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1733740905929 2024-12-09T10:41:47,567 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-09T10:41:47,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:47,567 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a9bc2b5e1505540f70456fbe90600f90#A#compaction#234 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:41:47,567 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing a9bc2b5e1505540f70456fbe90600f90 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-09T10:41:47,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9bc2b5e1505540f70456fbe90600f90, store=A 2024-12-09T10:41:47,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:41:47,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9bc2b5e1505540f70456fbe90600f90, store=B 2024-12-09T10:41:47,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:41:47,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9bc2b5e1505540f70456fbe90600f90, store=C 2024-12-09T10:41:47,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:41:47,568 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/A/92d349a3ea3e49e2bec79038736cc38f is 50, key is test_row_0/A:col10/1733740905929/Put/seqid=0 2024-12-09T10:41:47,571 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a9bc2b5e1505540f70456fbe90600f90#B#compaction#235 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:41:47,571 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/B/23d19da71fc34a67bb3e5357e6ccf5ae is 50, key is test_row_0/B:col10/1733740905929/Put/seqid=0 2024-12-09T10:41:47,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/A/918af91bd0dc43749111b7d0a4c56db8 is 50, key is test_row_0/A:col10/1733740906276/Put/seqid=0 2024-12-09T10:41:47,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742103_1279 (size=12595) 2024-12-09T10:41:47,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742104_1280 (size=12595) 2024-12-09T10:41:47,602 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/A/92d349a3ea3e49e2bec79038736cc38f as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/92d349a3ea3e49e2bec79038736cc38f 2024-12-09T10:41:47,610 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a9bc2b5e1505540f70456fbe90600f90/A of a9bc2b5e1505540f70456fbe90600f90 into 92d349a3ea3e49e2bec79038736cc38f(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:41:47,610 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a9bc2b5e1505540f70456fbe90600f90: 2024-12-09T10:41:47,610 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., storeName=a9bc2b5e1505540f70456fbe90600f90/A, priority=13, startTime=1733740907555; duration=0sec 2024-12-09T10:41:47,610 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T10:41:47,610 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9bc2b5e1505540f70456fbe90600f90:A 2024-12-09T10:41:47,610 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T10:41:47,611 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T10:41:47,611 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1541): a9bc2b5e1505540f70456fbe90600f90/C is initiating minor compaction (all files) 2024-12-09T10:41:47,611 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a9bc2b5e1505540f70456fbe90600f90/C in TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:47,611 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/a727e020dd764dc09c393485c0908644, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/9e88f495ee964ef4acf875676dfc1818, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/182b81033c434657ae315668105bfe40] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp, totalSize=35.9 K 2024-12-09T10:41:47,612 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting a727e020dd764dc09c393485c0908644, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1733740903089 2024-12-09T10:41:47,612 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9e88f495ee964ef4acf875676dfc1818, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1733740903732 2024-12-09T10:41:47,613 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 182b81033c434657ae315668105bfe40, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1733740905929 2024-12-09T10:41:47,614 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/B/23d19da71fc34a67bb3e5357e6ccf5ae as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/23d19da71fc34a67bb3e5357e6ccf5ae 2024-12-09T10:41:47,620 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a9bc2b5e1505540f70456fbe90600f90/B of a9bc2b5e1505540f70456fbe90600f90 into 23d19da71fc34a67bb3e5357e6ccf5ae(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:41:47,620 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a9bc2b5e1505540f70456fbe90600f90: 2024-12-09T10:41:47,620 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., storeName=a9bc2b5e1505540f70456fbe90600f90/B, priority=13, startTime=1733740907555; duration=0sec 2024-12-09T10:41:47,620 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:41:47,620 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9bc2b5e1505540f70456fbe90600f90:B 2024-12-09T10:41:47,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742105_1281 (size=12151) 2024-12-09T10:41:47,629 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=188 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/A/918af91bd0dc43749111b7d0a4c56db8 2024-12-09T10:41:47,630 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a9bc2b5e1505540f70456fbe90600f90#C#compaction#237 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:41:47,631 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/C/6cb94aee83fc4beb9c55273f274a82b2 is 50, key is test_row_0/C:col10/1733740905929/Put/seqid=0 2024-12-09T10:41:47,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/B/8ed89976b87d4de0b6d13592a65a621b is 50, key is test_row_0/B:col10/1733740906276/Put/seqid=0 2024-12-09T10:41:47,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742106_1282 (size=12595) 2024-12-09T10:41:47,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742107_1283 (size=12151) 2024-12-09T10:41:47,733 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=188 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/B/8ed89976b87d4de0b6d13592a65a621b 2024-12-09T10:41:47,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/C/16e767b674e24c1eacef216483c0e4cc is 50, key is test_row_0/C:col10/1733740906276/Put/seqid=0 2024-12-09T10:41:47,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742108_1284 (size=12151) 2024-12-09T10:41:48,110 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/C/6cb94aee83fc4beb9c55273f274a82b2 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/6cb94aee83fc4beb9c55273f274a82b2 2024-12-09T10:41:48,116 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a9bc2b5e1505540f70456fbe90600f90/C of a9bc2b5e1505540f70456fbe90600f90 into 6cb94aee83fc4beb9c55273f274a82b2(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:41:48,116 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a9bc2b5e1505540f70456fbe90600f90: 2024-12-09T10:41:48,116 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., storeName=a9bc2b5e1505540f70456fbe90600f90/C, priority=13, startTime=1733740907555; duration=0sec 2024-12-09T10:41:48,116 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:41:48,116 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9bc2b5e1505540f70456fbe90600f90:C 2024-12-09T10:41:48,180 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=188 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/C/16e767b674e24c1eacef216483c0e4cc 2024-12-09T10:41:48,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/A/918af91bd0dc43749111b7d0a4c56db8 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/918af91bd0dc43749111b7d0a4c56db8 2024-12-09T10:41:48,194 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/918af91bd0dc43749111b7d0a4c56db8, entries=150, sequenceid=188, filesize=11.9 K 2024-12-09T10:41:48,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/B/8ed89976b87d4de0b6d13592a65a621b as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/8ed89976b87d4de0b6d13592a65a621b 2024-12-09T10:41:48,201 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/8ed89976b87d4de0b6d13592a65a621b, entries=150, sequenceid=188, filesize=11.9 K 2024-12-09T10:41:48,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/C/16e767b674e24c1eacef216483c0e4cc as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/16e767b674e24c1eacef216483c0e4cc 2024-12-09T10:41:48,210 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/16e767b674e24c1eacef216483c0e4cc, entries=150, sequenceid=188, filesize=11.9 K 2024-12-09T10:41:48,212 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=0 B/0 for a9bc2b5e1505540f70456fbe90600f90 in 645ms, sequenceid=188, compaction requested=false 2024-12-09T10:41:48,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for a9bc2b5e1505540f70456fbe90600f90: 2024-12-09T10:41:48,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:48,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-12-09T10:41:48,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-12-09T10:41:48,218 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-12-09T10:41:48,218 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0300 sec 2024-12-09T10:41:48,222 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 2.0400 sec 2024-12-09T10:41:48,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-09T10:41:48,330 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T10:41:48,331 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-09T10:41:48,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-12-09T10:41:48,334 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T10:41:48,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-09T10:41:48,335 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T10:41:48,335 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T10:41:48,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-09T10:41:48,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(8855): Flush requested on a9bc2b5e1505540f70456fbe90600f90 2024-12-09T10:41:48,478 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a9bc2b5e1505540f70456fbe90600f90 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-09T10:41:48,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9bc2b5e1505540f70456fbe90600f90, store=A 2024-12-09T10:41:48,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:41:48,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9bc2b5e1505540f70456fbe90600f90, store=B 2024-12-09T10:41:48,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:41:48,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9bc2b5e1505540f70456fbe90600f90, store=C 2024-12-09T10:41:48,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:41:48,484 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/A/89076f62549c44eaab27987207b7307b is 50, key is test_row_0/A:col10/1733740908473/Put/seqid=0 2024-12-09T10:41:48,488 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-09T10:41:48,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:48,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. as already flushing 2024-12-09T10:41:48,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:48,488 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:48,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:48,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:48,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742109_1285 (size=12151) 2024-12-09T10:41:48,513 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:48,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50340 deadline: 1733740968508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:48,515 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733740908493, max timestamp=1733740908512Num uniques 859; 2024-12-09T10:41:48,515 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-09T10:41:48,515 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:48,516 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:48,516 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:48,527 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:48,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50390 deadline: 1733740968522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:48,527 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:48,527 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:48,527 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:48,535 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:48,535 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:48,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50422 deadline: 1733740968527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:48,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50410 deadline: 1733740968527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:48,535 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:48,536 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:48,536 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:48,536 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:48,536 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:48,536 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:48,623 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:48,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50340 deadline: 1733740968620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:48,624 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:48,624 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:48,624 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:48,634 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:48,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50390 deadline: 1733740968629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:48,634 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:48,634 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:48,634 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:48,645 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:48,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50410 deadline: 1733740968639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:48,645 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:48,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50422 deadline: 1733740968639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:48,645 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:48,645 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:48,645 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:48,645 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:48,645 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:48,645 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:48,649 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-09T10:41:48,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:48,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. as already flushing 2024-12-09T10:41:48,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:48,649 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:48,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:48,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-09T10:41:48,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:48,803 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-09T10:41:48,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:48,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. as already flushing 2024-12-09T10:41:48,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:48,804 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:48,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:48,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:48,832 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:48,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50340 deadline: 1733740968829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:48,832 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:48,832 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:48,832 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:48,843 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:48,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50390 deadline: 1733740968840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:48,844 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:48,844 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:48,844 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:48,853 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:48,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50410 deadline: 1733740968849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:48,853 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:48,853 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:48,854 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:48,854 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:48,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50422 deadline: 1733740968849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:48,854 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:48,854 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:48,854 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:48,897 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/A/89076f62549c44eaab27987207b7307b 2024-12-09T10:41:48,908 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/B/b061c9252d91485a958680a4aec5f63a is 50, key is test_row_0/B:col10/1733740908473/Put/seqid=0 2024-12-09T10:41:48,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742110_1286 (size=12151) 2024-12-09T10:41:48,920 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/B/b061c9252d91485a958680a4aec5f63a 2024-12-09T10:41:48,938 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/C/183c50b64f214704b7b923f9d14cbc4f is 50, key is test_row_0/C:col10/1733740908473/Put/seqid=0 2024-12-09T10:41:48,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742111_1287 (size=12151) 2024-12-09T10:41:48,957 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-09T10:41:48,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:48,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. as already flushing 2024-12-09T10:41:48,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:48,958 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:48,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:48,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:48,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-09T10:41:49,111 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-09T10:41:49,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:49,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. as already flushing 2024-12-09T10:41:49,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:49,112 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:49,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:49,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:49,144 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:49,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50340 deadline: 1733740969139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:49,144 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:49,144 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:49,144 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:49,154 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:49,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50390 deadline: 1733740969149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:49,154 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:49,154 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:49,154 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:49,166 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:49,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50422 deadline: 1733740969166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:49,167 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:49,167 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:49,167 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:49,169 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:49,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50410 deadline: 1733740969166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:49,170 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:49,170 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:49,170 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:49,265 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-09T10:41:49,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:49,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. as already flushing 2024-12-09T10:41:49,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:49,266 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:49,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:49,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:49,349 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/C/183c50b64f214704b7b923f9d14cbc4f 2024-12-09T10:41:49,354 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/A/89076f62549c44eaab27987207b7307b as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/89076f62549c44eaab27987207b7307b 2024-12-09T10:41:49,359 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/89076f62549c44eaab27987207b7307b, entries=150, sequenceid=202, filesize=11.9 K 2024-12-09T10:41:49,359 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(997): StoreScanner already has the close lock. There is no need to updateReaders 2024-12-09T10:41:49,360 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/B/b061c9252d91485a958680a4aec5f63a as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/b061c9252d91485a958680a4aec5f63a 2024-12-09T10:41:49,364 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/b061c9252d91485a958680a4aec5f63a, entries=150, sequenceid=202, filesize=11.9 K 2024-12-09T10:41:49,365 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/C/183c50b64f214704b7b923f9d14cbc4f as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/183c50b64f214704b7b923f9d14cbc4f 2024-12-09T10:41:49,370 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/183c50b64f214704b7b923f9d14cbc4f, entries=150, sequenceid=202, filesize=11.9 K 2024-12-09T10:41:49,371 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for a9bc2b5e1505540f70456fbe90600f90 in 893ms, sequenceid=202, compaction requested=true 2024-12-09T10:41:49,371 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a9bc2b5e1505540f70456fbe90600f90: 2024-12-09T10:41:49,372 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T10:41:49,372 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9bc2b5e1505540f70456fbe90600f90:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T10:41:49,372 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:41:49,372 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9bc2b5e1505540f70456fbe90600f90:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T10:41:49,372 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:41:49,372 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9bc2b5e1505540f70456fbe90600f90:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T10:41:49,372 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T10:41:49,372 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T10:41:49,374 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T10:41:49,374 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1541): a9bc2b5e1505540f70456fbe90600f90/B is initiating minor compaction (all files) 2024-12-09T10:41:49,374 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a9bc2b5e1505540f70456fbe90600f90/B in TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:49,374 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/23d19da71fc34a67bb3e5357e6ccf5ae, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/8ed89976b87d4de0b6d13592a65a621b, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/b061c9252d91485a958680a4aec5f63a] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp, totalSize=36.0 K 2024-12-09T10:41:49,374 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T10:41:49,374 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1541): a9bc2b5e1505540f70456fbe90600f90/A is initiating minor compaction (all files) 2024-12-09T10:41:49,374 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a9bc2b5e1505540f70456fbe90600f90/A in TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:49,374 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/92d349a3ea3e49e2bec79038736cc38f, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/918af91bd0dc43749111b7d0a4c56db8, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/89076f62549c44eaab27987207b7307b] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp, totalSize=36.0 K 2024-12-09T10:41:49,379 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 23d19da71fc34a67bb3e5357e6ccf5ae, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1733740905929 2024-12-09T10:41:49,379 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 92d349a3ea3e49e2bec79038736cc38f, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1733740905929 2024-12-09T10:41:49,380 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ed89976b87d4de0b6d13592a65a621b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1733740906270 2024-12-09T10:41:49,380 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 918af91bd0dc43749111b7d0a4c56db8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1733740906270 2024-12-09T10:41:49,380 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting b061c9252d91485a958680a4aec5f63a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1733740908451 2024-12-09T10:41:49,381 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 89076f62549c44eaab27987207b7307b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1733740908451 2024-12-09T10:41:49,389 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a9bc2b5e1505540f70456fbe90600f90#B#compaction#243 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:41:49,390 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/B/ef034b024da04d83964895652bdad55d is 50, key is test_row_0/B:col10/1733740908473/Put/seqid=0 2024-12-09T10:41:49,394 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a9bc2b5e1505540f70456fbe90600f90#A#compaction#244 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:41:49,394 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/A/928c531e88b4442089a3a56c3afa7565 is 50, key is test_row_0/A:col10/1733740908473/Put/seqid=0 2024-12-09T10:41:49,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742112_1288 (size=12697) 2024-12-09T10:41:49,405 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/B/ef034b024da04d83964895652bdad55d as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/ef034b024da04d83964895652bdad55d 2024-12-09T10:41:49,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742113_1289 (size=12697) 2024-12-09T10:41:49,411 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a9bc2b5e1505540f70456fbe90600f90/B of a9bc2b5e1505540f70456fbe90600f90 into ef034b024da04d83964895652bdad55d(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:41:49,411 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a9bc2b5e1505540f70456fbe90600f90: 2024-12-09T10:41:49,411 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., storeName=a9bc2b5e1505540f70456fbe90600f90/B, priority=13, startTime=1733740909372; duration=0sec 2024-12-09T10:41:49,411 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T10:41:49,411 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9bc2b5e1505540f70456fbe90600f90:B 2024-12-09T10:41:49,411 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T10:41:49,412 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T10:41:49,412 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1541): a9bc2b5e1505540f70456fbe90600f90/C is initiating minor compaction (all files) 2024-12-09T10:41:49,412 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a9bc2b5e1505540f70456fbe90600f90/C in TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:49,412 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/6cb94aee83fc4beb9c55273f274a82b2, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/16e767b674e24c1eacef216483c0e4cc, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/183c50b64f214704b7b923f9d14cbc4f] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp, totalSize=36.0 K 2024-12-09T10:41:49,413 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 6cb94aee83fc4beb9c55273f274a82b2, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1733740905929 2024-12-09T10:41:49,414 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 16e767b674e24c1eacef216483c0e4cc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1733740906270 2024-12-09T10:41:49,415 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 183c50b64f214704b7b923f9d14cbc4f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1733740908451 2024-12-09T10:41:49,416 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/A/928c531e88b4442089a3a56c3afa7565 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/928c531e88b4442089a3a56c3afa7565 2024-12-09T10:41:49,420 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-09T10:41:49,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:49,420 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing a9bc2b5e1505540f70456fbe90600f90 3/3 column families, dataSize=127.47 KB heapSize=371.88 KB 2024-12-09T10:41:49,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9bc2b5e1505540f70456fbe90600f90, store=A 2024-12-09T10:41:49,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T10:41:49,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9bc2b5e1505540f70456fbe90600f90, store=B 2024-12-09T10:41:49,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T10:41:49,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9bc2b5e1505540f70456fbe90600f90, store=C 2024-12-09T10:41:49,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T10:41:49,426 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a9bc2b5e1505540f70456fbe90600f90#C#compaction#245 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:41:49,426 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/C/521d3394712445018a340ba1021aebf2 is 50, key is test_row_0/C:col10/1733740908473/Put/seqid=0 2024-12-09T10:41:49,427 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a9bc2b5e1505540f70456fbe90600f90/A of a9bc2b5e1505540f70456fbe90600f90 into 928c531e88b4442089a3a56c3afa7565(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:41:49,427 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a9bc2b5e1505540f70456fbe90600f90: 2024-12-09T10:41:49,427 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., storeName=a9bc2b5e1505540f70456fbe90600f90/A, priority=13, startTime=1733740909372; duration=0sec 2024-12-09T10:41:49,428 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:41:49,428 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9bc2b5e1505540f70456fbe90600f90:A 2024-12-09T10:41:49,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/A/1f587f228c5d4b318ed3eed223fe3249 is 50, key is test_row_0/A:col10/1733740908525/Put/seqid=0 2024-12-09T10:41:49,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742114_1290 (size=12697) 2024-12-09T10:41:49,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742115_1291 (size=12151) 2024-12-09T10:41:49,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-09T10:41:49,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(8855): Flush requested on a9bc2b5e1505540f70456fbe90600f90 2024-12-09T10:41:49,653 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. as already flushing 2024-12-09T10:41:49,675 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:49,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50390 deadline: 1733740969670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:49,675 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:49,675 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:49,675 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:49,677 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:49,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50340 deadline: 1733740969674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:49,678 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:49,678 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:49,678 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:49,682 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:49,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50422 deadline: 1733740969679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:49,682 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:49,682 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:49,682 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:49,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:49,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50410 deadline: 1733740969680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:49,684 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:49,684 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:49,684 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:49,784 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:49,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50390 deadline: 1733740969780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:49,785 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:49,785 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:49,785 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:49,785 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:49,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50340 deadline: 1733740969780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:49,785 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:49,785 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:49,785 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:49,836 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/A/1f587f228c5d4b318ed3eed223fe3249 2024-12-09T10:41:49,839 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/C/521d3394712445018a340ba1021aebf2 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/521d3394712445018a340ba1021aebf2 2024-12-09T10:41:49,846 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a9bc2b5e1505540f70456fbe90600f90/C of a9bc2b5e1505540f70456fbe90600f90 into 521d3394712445018a340ba1021aebf2(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:41:49,846 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a9bc2b5e1505540f70456fbe90600f90: 2024-12-09T10:41:49,846 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., storeName=a9bc2b5e1505540f70456fbe90600f90/C, priority=13, startTime=1733740909372; duration=0sec 2024-12-09T10:41:49,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/B/62ad861ea4234cf983211efbc85edf9c is 50, key is test_row_0/B:col10/1733740908525/Put/seqid=0 2024-12-09T10:41:49,846 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:41:49,846 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9bc2b5e1505540f70456fbe90600f90:C 2024-12-09T10:41:49,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742116_1292 (size=12151) 2024-12-09T10:41:49,851 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/B/62ad861ea4234cf983211efbc85edf9c 2024-12-09T10:41:49,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/C/fa1cc8a0acb4472b8c6b3d81cc4dc647 is 50, key is test_row_0/C:col10/1733740908525/Put/seqid=0 2024-12-09T10:41:49,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742117_1293 (size=12151) 2024-12-09T10:41:49,877 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/C/fa1cc8a0acb4472b8c6b3d81cc4dc647 2024-12-09T10:41:49,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/A/1f587f228c5d4b318ed3eed223fe3249 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/1f587f228c5d4b318ed3eed223fe3249 2024-12-09T10:41:49,886 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/1f587f228c5d4b318ed3eed223fe3249, entries=150, sequenceid=229, filesize=11.9 K 2024-12-09T10:41:49,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/B/62ad861ea4234cf983211efbc85edf9c as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/62ad861ea4234cf983211efbc85edf9c 2024-12-09T10:41:49,892 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/62ad861ea4234cf983211efbc85edf9c, entries=150, sequenceid=229, filesize=11.9 K 2024-12-09T10:41:49,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/C/fa1cc8a0acb4472b8c6b3d81cc4dc647 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/fa1cc8a0acb4472b8c6b3d81cc4dc647 2024-12-09T10:41:49,899 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/fa1cc8a0acb4472b8c6b3d81cc4dc647, entries=150, sequenceid=229, filesize=11.9 K 2024-12-09T10:41:49,900 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~371.83 KB/380752, currentSize=46.96 KB/48090 for a9bc2b5e1505540f70456fbe90600f90 in 480ms, sequenceid=229, compaction requested=false 2024-12-09T10:41:49,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for a9bc2b5e1505540f70456fbe90600f90: 2024-12-09T10:41:49,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:49,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-12-09T10:41:49,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-12-09T10:41:49,913 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-12-09T10:41:49,913 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5680 sec 2024-12-09T10:41:49,916 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 1.5830 sec 2024-12-09T10:41:49,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(8855): Flush requested on a9bc2b5e1505540f70456fbe90600f90 2024-12-09T10:41:49,992 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a9bc2b5e1505540f70456fbe90600f90 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-09T10:41:49,992 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9bc2b5e1505540f70456fbe90600f90, store=A 2024-12-09T10:41:49,992 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:41:49,992 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9bc2b5e1505540f70456fbe90600f90, store=B 2024-12-09T10:41:49,992 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:41:49,992 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9bc2b5e1505540f70456fbe90600f90, store=C 2024-12-09T10:41:49,992 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:41:49,997 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/A/fe5cf388ae6f4d9ab5c2a40a6f12452d is 50, key is test_row_0/A:col10/1733740909990/Put/seqid=0 2024-12-09T10:41:50,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742118_1294 (size=14541) 2024-12-09T10:41:50,038 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:50,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50340 deadline: 1733740970033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:50,038 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:50,038 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:50,038 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:50,042 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:50,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50390 deadline: 1733740970038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:50,042 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:50,042 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:50,042 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:50,150 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:50,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50340 deadline: 1733740970149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:50,150 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:50,150 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:50,150 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:50,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50390 deadline: 1733740970149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:50,150 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:50,150 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:50,150 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:50,150 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:50,362 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:50,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50340 deadline: 1733740970360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:50,363 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:50,363 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:50,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50390 deadline: 1733740970360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:50,363 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:50,363 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:50,363 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:50,363 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:50,363 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:50,403 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/A/fe5cf388ae6f4d9ab5c2a40a6f12452d 2024-12-09T10:41:50,412 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/B/93900fb0c76f46ab8a704c71ada382ae is 50, key is test_row_0/B:col10/1733740909990/Put/seqid=0 2024-12-09T10:41:50,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742119_1295 (size=12151) 2024-12-09T10:41:50,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-09T10:41:50,481 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T10:41:50,481 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-09T10:41:50,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-12-09T10:41:50,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-09T10:41:50,485 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T10:41:50,486 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T10:41:50,486 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T10:41:50,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-09T10:41:50,640 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-09T10:41:50,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:50,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. as already flushing 2024-12-09T10:41:50,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:50,640 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:50,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:50,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:50,671 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:50,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50390 deadline: 1733740970669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:50,672 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:50,672 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:50,672 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:50,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50340 deadline: 1733740970669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:50,672 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:50,672 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:50,672 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:50,672 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:50,703 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:50,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50422 deadline: 1733740970700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:50,704 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:50,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50410 deadline: 1733740970700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:50,704 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 7ea43b39fc24,36101,1733740827678 for 'test_row_1' in a9bc2b5e1505540f70456fbe90600f90 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T10:41:50,705 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:50,705 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:50,705 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:50,704 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 7ea43b39fc24,36101,1733740827678 for 'test_row_0' in a9bc2b5e1505540f70456fbe90600f90 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T10:41:50,705 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:50,705 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:50,705 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:50,794 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-09T10:41:50,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:50,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. as already flushing 2024-12-09T10:41:50,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:50,794 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:50,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:50,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:50,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-09T10:41:50,817 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/B/93900fb0c76f46ab8a704c71ada382ae 2024-12-09T10:41:50,825 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/C/ecbe256b1a3c46afa9e34e78dd8b310d is 50, key is test_row_0/C:col10/1733740909990/Put/seqid=0 2024-12-09T10:41:50,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742120_1296 (size=12151) 2024-12-09T10:41:50,947 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-09T10:41:50,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:50,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. as already flushing 2024-12-09T10:41:50,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:50,948 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:50,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:50,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:51,101 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-09T10:41:51,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:51,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. as already flushing 2024-12-09T10:41:51,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:51,101 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:51,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:51,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:51,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-09T10:41:51,181 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:51,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50390 deadline: 1733740971180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:51,182 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:51,182 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:51,182 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:51,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50340 deadline: 1733740971180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:51,182 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:51,182 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:51,182 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:51,182 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:51,231 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/C/ecbe256b1a3c46afa9e34e78dd8b310d 2024-12-09T10:41:51,238 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/A/fe5cf388ae6f4d9ab5c2a40a6f12452d as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/fe5cf388ae6f4d9ab5c2a40a6f12452d 2024-12-09T10:41:51,242 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/fe5cf388ae6f4d9ab5c2a40a6f12452d, entries=200, sequenceid=243, filesize=14.2 K 2024-12-09T10:41:51,243 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/B/93900fb0c76f46ab8a704c71ada382ae as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/93900fb0c76f46ab8a704c71ada382ae 2024-12-09T10:41:51,246 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/93900fb0c76f46ab8a704c71ada382ae, entries=150, sequenceid=243, filesize=11.9 K 2024-12-09T10:41:51,247 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/C/ecbe256b1a3c46afa9e34e78dd8b310d as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/ecbe256b1a3c46afa9e34e78dd8b310d 2024-12-09T10:41:51,252 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/ecbe256b1a3c46afa9e34e78dd8b310d, entries=150, sequenceid=243, filesize=11.9 K 2024-12-09T10:41:51,252 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for a9bc2b5e1505540f70456fbe90600f90 in 1260ms, sequenceid=243, compaction requested=true 2024-12-09T10:41:51,252 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a9bc2b5e1505540f70456fbe90600f90: 2024-12-09T10:41:51,253 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9bc2b5e1505540f70456fbe90600f90:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T10:41:51,253 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:41:51,253 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T10:41:51,253 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9bc2b5e1505540f70456fbe90600f90:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T10:41:51,253 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:41:51,253 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9bc2b5e1505540f70456fbe90600f90:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T10:41:51,253 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T10:41:51,253 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T10:41:51,253 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-09T10:41:51,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:51,254 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing a9bc2b5e1505540f70456fbe90600f90 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-09T10:41:51,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9bc2b5e1505540f70456fbe90600f90, store=A 2024-12-09T10:41:51,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:41:51,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9bc2b5e1505540f70456fbe90600f90, store=B 2024-12-09T10:41:51,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:41:51,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9bc2b5e1505540f70456fbe90600f90, store=C 2024-12-09T10:41:51,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:41:51,256 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39389 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T10:41:51,256 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36999 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T10:41:51,256 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1541): a9bc2b5e1505540f70456fbe90600f90/A is initiating minor compaction (all files) 2024-12-09T10:41:51,256 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1541): a9bc2b5e1505540f70456fbe90600f90/B is initiating minor compaction (all files) 2024-12-09T10:41:51,256 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a9bc2b5e1505540f70456fbe90600f90/A in TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:51,256 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a9bc2b5e1505540f70456fbe90600f90/B in TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:51,256 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/928c531e88b4442089a3a56c3afa7565, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/1f587f228c5d4b318ed3eed223fe3249, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/fe5cf388ae6f4d9ab5c2a40a6f12452d] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp, totalSize=38.5 K 2024-12-09T10:41:51,256 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/ef034b024da04d83964895652bdad55d, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/62ad861ea4234cf983211efbc85edf9c, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/93900fb0c76f46ab8a704c71ada382ae] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp, totalSize=36.1 K 2024-12-09T10:41:51,257 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 928c531e88b4442089a3a56c3afa7565, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1733740908451 2024-12-09T10:41:51,257 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting ef034b024da04d83964895652bdad55d, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1733740908451 2024-12-09T10:41:51,257 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 62ad861ea4234cf983211efbc85edf9c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1733740908520 2024-12-09T10:41:51,258 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1f587f228c5d4b318ed3eed223fe3249, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1733740908520 2024-12-09T10:41:51,258 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting fe5cf388ae6f4d9ab5c2a40a6f12452d, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1733740909663 2024-12-09T10:41:51,258 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 93900fb0c76f46ab8a704c71ada382ae, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1733740909663 2024-12-09T10:41:51,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/A/29f767df228e4b0998b8e64c76fc7481 is 50, key is test_row_0/A:col10/1733740910030/Put/seqid=0 2024-12-09T10:41:51,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742121_1297 (size=12301) 2024-12-09T10:41:51,279 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a9bc2b5e1505540f70456fbe90600f90#B#compaction#253 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:41:51,280 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/B/1a2d950d62634c5a9a0b75c24c59ba09 is 50, key is test_row_0/B:col10/1733740909990/Put/seqid=0 2024-12-09T10:41:51,288 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a9bc2b5e1505540f70456fbe90600f90#A#compaction#254 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:41:51,289 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/A/fefc428ace66468f8584f0f2b5e64fd7 is 50, key is test_row_0/A:col10/1733740909990/Put/seqid=0 2024-12-09T10:41:51,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742122_1298 (size=12799) 2024-12-09T10:41:51,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742123_1299 (size=12799) 2024-12-09T10:41:51,307 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/A/fefc428ace66468f8584f0f2b5e64fd7 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/fefc428ace66468f8584f0f2b5e64fd7 2024-12-09T10:41:51,313 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a9bc2b5e1505540f70456fbe90600f90/A of a9bc2b5e1505540f70456fbe90600f90 into fefc428ace66468f8584f0f2b5e64fd7(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:41:51,313 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a9bc2b5e1505540f70456fbe90600f90: 2024-12-09T10:41:51,313 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., storeName=a9bc2b5e1505540f70456fbe90600f90/A, priority=13, startTime=1733740911253; duration=0sec 2024-12-09T10:41:51,313 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T10:41:51,313 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9bc2b5e1505540f70456fbe90600f90:A 2024-12-09T10:41:51,313 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T10:41:51,316 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36999 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T10:41:51,316 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1541): a9bc2b5e1505540f70456fbe90600f90/C is initiating minor compaction (all files) 2024-12-09T10:41:51,316 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a9bc2b5e1505540f70456fbe90600f90/C in TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:51,316 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/521d3394712445018a340ba1021aebf2, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/fa1cc8a0acb4472b8c6b3d81cc4dc647, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/ecbe256b1a3c46afa9e34e78dd8b310d] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp, totalSize=36.1 K 2024-12-09T10:41:51,317 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 521d3394712445018a340ba1021aebf2, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1733740908451 2024-12-09T10:41:51,317 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting fa1cc8a0acb4472b8c6b3d81cc4dc647, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1733740908520 2024-12-09T10:41:51,317 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting ecbe256b1a3c46afa9e34e78dd8b310d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1733740909663 2024-12-09T10:41:51,329 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a9bc2b5e1505540f70456fbe90600f90#C#compaction#255 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:41:51,329 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/C/e8fa9f75e7864d0b827cdc9d93bdb516 is 50, key is test_row_0/C:col10/1733740909990/Put/seqid=0 2024-12-09T10:41:51,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742124_1300 (size=12799) 2024-12-09T10:41:51,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-09T10:41:51,669 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/A/29f767df228e4b0998b8e64c76fc7481 2024-12-09T10:41:51,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/B/321fba8eefe64b88be59a433c76d8f02 is 50, key is test_row_0/B:col10/1733740910030/Put/seqid=0 2024-12-09T10:41:51,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742125_1301 (size=12301) 2024-12-09T10:41:51,706 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/B/1a2d950d62634c5a9a0b75c24c59ba09 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/1a2d950d62634c5a9a0b75c24c59ba09 2024-12-09T10:41:51,711 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a9bc2b5e1505540f70456fbe90600f90/B of a9bc2b5e1505540f70456fbe90600f90 into 1a2d950d62634c5a9a0b75c24c59ba09(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:41:51,711 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a9bc2b5e1505540f70456fbe90600f90: 2024-12-09T10:41:51,711 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., storeName=a9bc2b5e1505540f70456fbe90600f90/B, priority=13, startTime=1733740911253; duration=0sec 2024-12-09T10:41:51,712 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:41:51,712 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9bc2b5e1505540f70456fbe90600f90:B 2024-12-09T10:41:51,748 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/C/e8fa9f75e7864d0b827cdc9d93bdb516 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/e8fa9f75e7864d0b827cdc9d93bdb516 2024-12-09T10:41:51,753 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a9bc2b5e1505540f70456fbe90600f90/C of a9bc2b5e1505540f70456fbe90600f90 into e8fa9f75e7864d0b827cdc9d93bdb516(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:41:51,753 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a9bc2b5e1505540f70456fbe90600f90: 2024-12-09T10:41:51,753 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., storeName=a9bc2b5e1505540f70456fbe90600f90/C, priority=13, startTime=1733740911253; duration=0sec 2024-12-09T10:41:51,753 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:41:51,753 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9bc2b5e1505540f70456fbe90600f90:C 2024-12-09T10:41:52,093 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/B/321fba8eefe64b88be59a433c76d8f02 2024-12-09T10:41:52,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/C/3a959d9babc44ee7977ecd144ce50d32 is 50, key is test_row_0/C:col10/1733740910030/Put/seqid=0 2024-12-09T10:41:52,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742126_1302 (size=12301) 2024-12-09T10:41:52,135 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/C/3a959d9babc44ee7977ecd144ce50d32 2024-12-09T10:41:52,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/A/29f767df228e4b0998b8e64c76fc7481 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/29f767df228e4b0998b8e64c76fc7481 2024-12-09T10:41:52,155 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/29f767df228e4b0998b8e64c76fc7481, entries=150, sequenceid=263, filesize=12.0 K 2024-12-09T10:41:52,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/B/321fba8eefe64b88be59a433c76d8f02 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/321fba8eefe64b88be59a433c76d8f02 2024-12-09T10:41:52,167 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/321fba8eefe64b88be59a433c76d8f02, entries=150, sequenceid=263, filesize=12.0 K 2024-12-09T10:41:52,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/C/3a959d9babc44ee7977ecd144ce50d32 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/3a959d9babc44ee7977ecd144ce50d32 2024-12-09T10:41:52,175 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/3a959d9babc44ee7977ecd144ce50d32, entries=150, sequenceid=263, filesize=12.0 K 2024-12-09T10:41:52,178 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=0 B/0 for a9bc2b5e1505540f70456fbe90600f90 in 924ms, sequenceid=263, compaction requested=false 2024-12-09T10:41:52,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for a9bc2b5e1505540f70456fbe90600f90: 2024-12-09T10:41:52,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:52,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-12-09T10:41:52,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-12-09T10:41:52,184 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-12-09T10:41:52,184 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6950 sec 2024-12-09T10:41:52,187 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 1.7040 sec 2024-12-09T10:41:52,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(8855): Flush requested on a9bc2b5e1505540f70456fbe90600f90 2024-12-09T10:41:52,211 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a9bc2b5e1505540f70456fbe90600f90 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-09T10:41:52,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9bc2b5e1505540f70456fbe90600f90, store=A 2024-12-09T10:41:52,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:41:52,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9bc2b5e1505540f70456fbe90600f90, store=B 2024-12-09T10:41:52,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:41:52,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9bc2b5e1505540f70456fbe90600f90, store=C 2024-12-09T10:41:52,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:41:52,218 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/A/91291059081a40259f14082260b0e316 is 50, key is test_row_0/A:col10/1733740912206/Put/seqid=0 2024-12-09T10:41:52,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742127_1303 (size=14741) 2024-12-09T10:41:52,252 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/A/91291059081a40259f14082260b0e316 2024-12-09T10:41:52,272 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/B/8a408e3b15d1482686ddedea512cc68d is 50, key is test_row_0/B:col10/1733740912206/Put/seqid=0 2024-12-09T10:41:52,280 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733740912235, max timestamp=1733740912272Num uniques 859; 2024-12-09T10:41:52,280 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733740912235, max timestamp=1733740912272Num uniques 859; 2024-12-09T10:41:52,280 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T10:41:52,280 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T10:41:52,283 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733740912235, max timestamp=1733740912272Num uniques 859; 2024-12-09T10:41:52,283 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T10:41:52,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742128_1304 (size=12301) 2024-12-09T10:41:52,343 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:52,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50340 deadline: 1733740972338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:52,343 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:52,344 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:52,344 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:52,345 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:52,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50390 deadline: 1733740972339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:52,346 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:52,346 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:52,346 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:52,455 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:52,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50340 deadline: 1733740972450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:52,455 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:52,455 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:52,456 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:52,456 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:52,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50390 deadline: 1733740972450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:52,456 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:52,456 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:52,456 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:52,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-09T10:41:52,630 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T10:41:52,631 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-09T10:41:52,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-12-09T10:41:52,634 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T10:41:52,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-09T10:41:52,636 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T10:41:52,636 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T10:41:52,664 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:52,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50340 deadline: 1733740972660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:52,664 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:52,664 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:52,664 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:52,664 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:52,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50390 deadline: 1733740972660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:52,665 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:52,665 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:52,665 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:52,722 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:52,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50422 deadline: 1733740972720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:52,722 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/B/8a408e3b15d1482686ddedea512cc68d 2024-12-09T10:41:52,723 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 7ea43b39fc24,36101,1733740827678 for 'test_row_0' in a9bc2b5e1505540f70456fbe90600f90 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T10:41:52,723 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:52,724 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:52,724 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:52,732 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:52,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50410 deadline: 1733740972730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:52,733 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 7ea43b39fc24,36101,1733740827678 for 'test_row_1' in a9bc2b5e1505540f70456fbe90600f90 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4206 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T10:41:52,734 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:52,734 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:52,734 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:52,738 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/C/796fadd36d244fa5ab03cf8da7e0a43c is 50, key is test_row_0/C:col10/1733740912206/Put/seqid=0 2024-12-09T10:41:52,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-09T10:41:52,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742129_1305 (size=12301) 2024-12-09T10:41:52,758 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/C/796fadd36d244fa5ab03cf8da7e0a43c 2024-12-09T10:41:52,764 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/A/91291059081a40259f14082260b0e316 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/91291059081a40259f14082260b0e316 2024-12-09T10:41:52,769 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/91291059081a40259f14082260b0e316, entries=200, sequenceid=277, filesize=14.4 K 2024-12-09T10:41:52,773 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/B/8a408e3b15d1482686ddedea512cc68d as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/8a408e3b15d1482686ddedea512cc68d 2024-12-09T10:41:52,782 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/8a408e3b15d1482686ddedea512cc68d, entries=150, sequenceid=277, filesize=12.0 K 2024-12-09T10:41:52,785 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/C/796fadd36d244fa5ab03cf8da7e0a43c as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/796fadd36d244fa5ab03cf8da7e0a43c 2024-12-09T10:41:52,789 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-09T10:41:52,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:52,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. as already flushing 2024-12-09T10:41:52,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:52,789 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:52,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:52,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:52,794 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/796fadd36d244fa5ab03cf8da7e0a43c, entries=150, sequenceid=277, filesize=12.0 K 2024-12-09T10:41:52,795 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for a9bc2b5e1505540f70456fbe90600f90 in 584ms, sequenceid=277, compaction requested=true 2024-12-09T10:41:52,795 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a9bc2b5e1505540f70456fbe90600f90: 2024-12-09T10:41:52,795 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T10:41:52,795 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9bc2b5e1505540f70456fbe90600f90:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T10:41:52,796 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:41:52,796 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T10:41:52,796 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9bc2b5e1505540f70456fbe90600f90:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T10:41:52,796 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:41:52,796 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9bc2b5e1505540f70456fbe90600f90:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T10:41:52,796 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T10:41:52,797 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37401 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T10:41:52,797 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1541): a9bc2b5e1505540f70456fbe90600f90/B is initiating minor compaction (all files) 2024-12-09T10:41:52,797 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a9bc2b5e1505540f70456fbe90600f90/B in TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:52,797 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/1a2d950d62634c5a9a0b75c24c59ba09, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/321fba8eefe64b88be59a433c76d8f02, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/8a408e3b15d1482686ddedea512cc68d] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp, totalSize=36.5 K 2024-12-09T10:41:52,797 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39841 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T10:41:52,797 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1541): a9bc2b5e1505540f70456fbe90600f90/A is initiating minor compaction (all files) 2024-12-09T10:41:52,798 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a9bc2b5e1505540f70456fbe90600f90/A in TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:52,798 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/fefc428ace66468f8584f0f2b5e64fd7, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/29f767df228e4b0998b8e64c76fc7481, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/91291059081a40259f14082260b0e316] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp, totalSize=38.9 K 2024-12-09T10:41:52,806 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting fefc428ace66468f8584f0f2b5e64fd7, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1733740909663 2024-12-09T10:41:52,806 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 1a2d950d62634c5a9a0b75c24c59ba09, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1733740909663 2024-12-09T10:41:52,807 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 321fba8eefe64b88be59a433c76d8f02, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1733740910024 2024-12-09T10:41:52,807 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 29f767df228e4b0998b8e64c76fc7481, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1733740910024 2024-12-09T10:41:52,808 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a408e3b15d1482686ddedea512cc68d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1733740912206 2024-12-09T10:41:52,808 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 91291059081a40259f14082260b0e316, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1733740912205 2024-12-09T10:41:52,820 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a9bc2b5e1505540f70456fbe90600f90#B#compaction#261 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:41:52,820 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/B/3eca2fe3703d4d6eb24d49f1ff3a2395 is 50, key is test_row_0/B:col10/1733740912206/Put/seqid=0 2024-12-09T10:41:52,825 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a9bc2b5e1505540f70456fbe90600f90#A#compaction#262 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:41:52,825 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/A/093db4477dab446288957c279bb5f1c4 is 50, key is test_row_0/A:col10/1733740912206/Put/seqid=0 2024-12-09T10:41:52,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742130_1306 (size=13051) 2024-12-09T10:41:52,849 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/B/3eca2fe3703d4d6eb24d49f1ff3a2395 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/3eca2fe3703d4d6eb24d49f1ff3a2395 2024-12-09T10:41:52,861 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a9bc2b5e1505540f70456fbe90600f90/B of a9bc2b5e1505540f70456fbe90600f90 into 3eca2fe3703d4d6eb24d49f1ff3a2395(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:41:52,861 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a9bc2b5e1505540f70456fbe90600f90: 2024-12-09T10:41:52,861 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., storeName=a9bc2b5e1505540f70456fbe90600f90/B, priority=13, startTime=1733740912796; duration=0sec 2024-12-09T10:41:52,861 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T10:41:52,861 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9bc2b5e1505540f70456fbe90600f90:B 2024-12-09T10:41:52,861 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T10:41:52,863 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37401 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T10:41:52,863 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1541): a9bc2b5e1505540f70456fbe90600f90/C is initiating minor compaction (all files) 2024-12-09T10:41:52,863 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a9bc2b5e1505540f70456fbe90600f90/C in TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:52,863 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/e8fa9f75e7864d0b827cdc9d93bdb516, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/3a959d9babc44ee7977ecd144ce50d32, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/796fadd36d244fa5ab03cf8da7e0a43c] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp, totalSize=36.5 K 2024-12-09T10:41:52,864 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting e8fa9f75e7864d0b827cdc9d93bdb516, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1733740909663 2024-12-09T10:41:52,864 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a959d9babc44ee7977ecd144ce50d32, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1733740910024 2024-12-09T10:41:52,864 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 796fadd36d244fa5ab03cf8da7e0a43c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1733740912206 2024-12-09T10:41:52,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742131_1307 (size=13051) 2024-12-09T10:41:52,873 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a9bc2b5e1505540f70456fbe90600f90#C#compaction#263 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:41:52,873 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/C/c15fa0fb2b3f4478b2769fa9501dbe82 is 50, key is test_row_0/C:col10/1733740912206/Put/seqid=0 2024-12-09T10:41:52,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742132_1308 (size=13051) 2024-12-09T10:41:52,944 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-09T10:41:52,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:52,945 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing a9bc2b5e1505540f70456fbe90600f90 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-12-09T10:41:52,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9bc2b5e1505540f70456fbe90600f90, store=A 2024-12-09T10:41:52,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T10:41:52,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9bc2b5e1505540f70456fbe90600f90, store=B 2024-12-09T10:41:52,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T10:41:52,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9bc2b5e1505540f70456fbe90600f90, store=C 2024-12-09T10:41:52,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T10:41:52,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-09T10:41:52,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/A/9f7e54c866b4411084e1aa97efa573c3 is 50, key is test_row_0/A:col10/1733740912336/Put/seqid=0 2024-12-09T10:41:52,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742133_1309 (size=12301) 2024-12-09T10:41:52,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(8855): Flush requested on a9bc2b5e1505540f70456fbe90600f90 2024-12-09T10:41:52,975 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. as already flushing 2024-12-09T10:41:52,999 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:52,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50340 deadline: 1733740972994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:52,999 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:52,999 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:52,999 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:53,000 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:53,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50390 deadline: 1733740972995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:53,001 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:53,001 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:53,001 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:53,114 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:53,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50340 deadline: 1733740973110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:53,114 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:53,114 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:53,114 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:53,114 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:53,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50390 deadline: 1733740973110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:53,115 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:53,115 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:53,115 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:53,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-09T10:41:53,268 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/A/093db4477dab446288957c279bb5f1c4 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/093db4477dab446288957c279bb5f1c4 2024-12-09T10:41:53,273 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a9bc2b5e1505540f70456fbe90600f90/A of a9bc2b5e1505540f70456fbe90600f90 into 093db4477dab446288957c279bb5f1c4(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:41:53,273 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a9bc2b5e1505540f70456fbe90600f90: 2024-12-09T10:41:53,274 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., storeName=a9bc2b5e1505540f70456fbe90600f90/A, priority=13, startTime=1733740912795; duration=0sec 2024-12-09T10:41:53,274 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:41:53,274 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9bc2b5e1505540f70456fbe90600f90:A 2024-12-09T10:41:53,288 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/C/c15fa0fb2b3f4478b2769fa9501dbe82 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/c15fa0fb2b3f4478b2769fa9501dbe82 2024-12-09T10:41:53,293 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a9bc2b5e1505540f70456fbe90600f90/C of a9bc2b5e1505540f70456fbe90600f90 into c15fa0fb2b3f4478b2769fa9501dbe82(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:41:53,293 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a9bc2b5e1505540f70456fbe90600f90: 2024-12-09T10:41:53,293 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., storeName=a9bc2b5e1505540f70456fbe90600f90/C, priority=13, startTime=1733740912796; duration=0sec 2024-12-09T10:41:53,293 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:41:53,293 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9bc2b5e1505540f70456fbe90600f90:C 2024-12-09T10:41:53,323 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:53,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50340 deadline: 1733740973319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:53,323 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:53,323 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:53,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50390 deadline: 1733740973319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:53,323 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:53,323 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:53,323 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:53,324 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:53,324 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:53,357 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/A/9f7e54c866b4411084e1aa97efa573c3 2024-12-09T10:41:53,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/B/a89387fe7ef14e278012eb283087a4c0 is 50, key is test_row_0/B:col10/1733740912336/Put/seqid=0 2024-12-09T10:41:53,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742134_1310 (size=12301) 2024-12-09T10:41:53,631 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:53,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50340 deadline: 1733740973629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:53,632 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:53,632 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:53,632 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:53,632 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:53,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50390 deadline: 1733740973629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:53,632 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:53,632 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:53,632 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:53,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-09T10:41:53,772 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/B/a89387fe7ef14e278012eb283087a4c0 2024-12-09T10:41:53,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/C/07242d4f52b749789eb5b02297f6ec71 is 50, key is test_row_0/C:col10/1733740912336/Put/seqid=0 2024-12-09T10:41:53,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742135_1311 (size=12301) 2024-12-09T10:41:54,144 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:54,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50340 deadline: 1733740974139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:54,144 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:54,144 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:54,144 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:54,144 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:54,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50390 deadline: 1733740974139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:54,145 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:54,145 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:54,145 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:54,185 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/C/07242d4f52b749789eb5b02297f6ec71 2024-12-09T10:41:54,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/A/9f7e54c866b4411084e1aa97efa573c3 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/9f7e54c866b4411084e1aa97efa573c3 2024-12-09T10:41:54,193 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/9f7e54c866b4411084e1aa97efa573c3, entries=150, sequenceid=314, filesize=12.0 K 2024-12-09T10:41:54,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/B/a89387fe7ef14e278012eb283087a4c0 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/a89387fe7ef14e278012eb283087a4c0 2024-12-09T10:41:54,198 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/a89387fe7ef14e278012eb283087a4c0, entries=150, sequenceid=314, filesize=12.0 K 2024-12-09T10:41:54,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/C/07242d4f52b749789eb5b02297f6ec71 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/07242d4f52b749789eb5b02297f6ec71 2024-12-09T10:41:54,203 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/07242d4f52b749789eb5b02297f6ec71, entries=150, sequenceid=314, filesize=12.0 K 2024-12-09T10:41:54,204 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for a9bc2b5e1505540f70456fbe90600f90 in 1259ms, sequenceid=314, compaction requested=false 2024-12-09T10:41:54,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for a9bc2b5e1505540f70456fbe90600f90: 2024-12-09T10:41:54,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:54,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-12-09T10:41:54,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-12-09T10:41:54,208 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-12-09T10:41:54,208 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5700 sec 2024-12-09T10:41:54,210 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 1.5770 sec 2024-12-09T10:41:54,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-09T10:41:54,780 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T10:41:54,780 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-09T10:41:54,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-12-09T10:41:54,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-09T10:41:54,783 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T10:41:54,783 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T10:41:54,784 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T10:41:54,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-09T10:41:54,936 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-09T10:41:54,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:54,937 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing a9bc2b5e1505540f70456fbe90600f90 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-09T10:41:54,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9bc2b5e1505540f70456fbe90600f90, store=A 2024-12-09T10:41:54,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:41:54,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9bc2b5e1505540f70456fbe90600f90, store=B 2024-12-09T10:41:54,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:41:54,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9bc2b5e1505540f70456fbe90600f90, store=C 2024-12-09T10:41:54,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:41:54,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/A/320c0829b8f54f869b1023ae2ebebce8 is 50, key is test_row_0/A:col10/1733740912980/Put/seqid=0 2024-12-09T10:41:54,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742136_1312 (size=12301) 2024-12-09T10:41:55,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-09T10:41:55,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(8855): Flush requested on a9bc2b5e1505540f70456fbe90600f90 2024-12-09T10:41:55,162 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. as already flushing 2024-12-09T10:41:55,212 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733740915195, max timestamp=1733740915208Num uniques 859; 2024-12-09T10:41:55,212 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T10:41:55,212 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733740915195, max timestamp=1733740915208Num uniques 859; 2024-12-09T10:41:55,212 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733740915195, max timestamp=1733740915208Num uniques 859; 2024-12-09T10:41:55,212 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T10:41:55,212 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T10:41:55,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:55,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50340 deadline: 1733740975253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:55,258 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:55,258 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:55,259 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:55,260 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:55,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50390 deadline: 1733740975255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:55,261 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:55,261 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:55,261 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:55,351 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=327 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/A/320c0829b8f54f869b1023ae2ebebce8 2024-12-09T10:41:55,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/B/ff9bd9d6e35f45109139d5b08f7e2c38 is 50, key is test_row_0/B:col10/1733740912980/Put/seqid=0 2024-12-09T10:41:55,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742137_1313 (size=12301) 2024-12-09T10:41:55,364 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=327 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/B/ff9bd9d6e35f45109139d5b08f7e2c38 2024-12-09T10:41:55,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/C/8896a3f96c3046cd86cd3de1ab0e4551 is 50, key is test_row_0/C:col10/1733740912980/Put/seqid=0 2024-12-09T10:41:55,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742138_1314 (size=12301) 2024-12-09T10:41:55,375 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:55,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50340 deadline: 1733740975370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:55,375 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:55,375 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:55,375 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:55,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50390 deadline: 1733740975372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:55,375 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:55,376 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:55,376 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:55,376 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:55,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-09T10:41:55,584 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:55,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50340 deadline: 1733740975580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:55,584 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:55,584 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:55,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50390 deadline: 1733740975580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:55,584 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:55,585 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:55,585 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:55,585 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:55,585 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:55,775 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=327 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/C/8896a3f96c3046cd86cd3de1ab0e4551 2024-12-09T10:41:55,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/A/320c0829b8f54f869b1023ae2ebebce8 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/320c0829b8f54f869b1023ae2ebebce8 2024-12-09T10:41:55,784 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/320c0829b8f54f869b1023ae2ebebce8, entries=150, sequenceid=327, filesize=12.0 K 2024-12-09T10:41:55,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/B/ff9bd9d6e35f45109139d5b08f7e2c38 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/ff9bd9d6e35f45109139d5b08f7e2c38 2024-12-09T10:41:55,789 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/ff9bd9d6e35f45109139d5b08f7e2c38, entries=150, sequenceid=327, filesize=12.0 K 2024-12-09T10:41:55,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/C/8896a3f96c3046cd86cd3de1ab0e4551 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/8896a3f96c3046cd86cd3de1ab0e4551 2024-12-09T10:41:55,793 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/8896a3f96c3046cd86cd3de1ab0e4551, entries=150, sequenceid=327, filesize=12.0 K 2024-12-09T10:41:55,794 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for a9bc2b5e1505540f70456fbe90600f90 in 858ms, sequenceid=327, compaction requested=true 2024-12-09T10:41:55,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for a9bc2b5e1505540f70456fbe90600f90: 2024-12-09T10:41:55,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:55,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-12-09T10:41:55,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-12-09T10:41:55,797 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-12-09T10:41:55,797 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0120 sec 2024-12-09T10:41:55,799 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 1.0170 sec 2024-12-09T10:41:55,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(8855): Flush requested on a9bc2b5e1505540f70456fbe90600f90 2024-12-09T10:41:55,895 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a9bc2b5e1505540f70456fbe90600f90 3/3 column families, dataSize=134.18 KB heapSize=385.22 KB 2024-12-09T10:41:55,900 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733740915241, max timestamp=1733740915894Num uniques 1018; 2024-12-09T10:41:55,900 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733740915241, max timestamp=1733740915894Num uniques 1018; 2024-12-09T10:41:55,900 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733740915241, max timestamp=1733740915894Num uniques 1018; 2024-12-09T10:41:55,900 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-09T10:41:55,900 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-09T10:41:55,900 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-09T10:41:55,900 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9bc2b5e1505540f70456fbe90600f90, store=A 2024-12-09T10:41:55,900 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T10:41:55,900 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9bc2b5e1505540f70456fbe90600f90, store=B 2024-12-09T10:41:55,900 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T10:41:55,900 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9bc2b5e1505540f70456fbe90600f90, store=C 2024-12-09T10:41:55,900 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T10:41:55,905 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/A/753cc2ad7431459fbc209c1bc6714a95 is 50, key is test_row_0/A:col10/1733740915247/Put/seqid=0 2024-12-09T10:41:55,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742139_1315 (size=12301) 2024-12-09T10:41:55,912 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/A/753cc2ad7431459fbc209c1bc6714a95 2024-12-09T10:41:55,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-09T10:41:55,919 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T10:41:55,920 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/B/b55f1eb557544697ab9b49f2641e2c30 is 50, key is test_row_0/B:col10/1733740915247/Put/seqid=0 2024-12-09T10:41:55,922 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-09T10:41:55,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-12-09T10:41:55,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-09T10:41:55,924 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T10:41:55,925 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T10:41:55,925 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T10:41:55,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742140_1316 (size=12301) 2024-12-09T10:41:55,928 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/B/b55f1eb557544697ab9b49f2641e2c30 2024-12-09T10:41:55,933 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-09T10:41:55,935 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/C/c803a4d7149546b0a1302fdd47e8790d is 50, key is test_row_0/C:col10/1733740915247/Put/seqid=0 2024-12-09T10:41:55,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742141_1317 (size=12301) 2024-12-09T10:41:55,952 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733740915937, max timestamp=1733740915949Num uniques 859; 2024-12-09T10:41:55,952 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733740915937, max timestamp=1733740915949Num uniques 859; 2024-12-09T10:41:55,952 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T10:41:55,952 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T10:41:55,952 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733740915937, max timestamp=1733740915949Num uniques 859; 2024-12-09T10:41:55,952 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T10:41:56,010 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7680, totalHeapSize=17712, min timestamp=1733740915993, max timestamp=1733740916005Num uniques 1068; 2024-12-09T10:41:56,010 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48900, new segment data size=7680, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-12-09T10:41:56,010 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733740915993, max timestamp=1733740916005Num uniques 1068; 2024-12-09T10:41:56,010 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7683, totalHeapSize=17712, min timestamp=1733740915993, max timestamp=1733740916005Num uniques 1068; 2024-12-09T10:41:56,010 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-12-09T10:41:56,010 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48903, new segment data size=7683, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-12-09T10:41:56,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-09T10:41:56,061 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733740916028, max timestamp=1733740916057Num uniques 1027; 2024-12-09T10:41:56,061 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-12-09T10:41:56,061 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733740916028, max timestamp=1733740916057Num uniques 1027; 2024-12-09T10:41:56,061 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733740916028, max timestamp=1733740916057Num uniques 1027; 2024-12-09T10:41:56,061 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-12-09T10:41:56,061 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-12-09T10:41:56,079 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-09T10:41:56,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:56,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. as already flushing 2024-12-09T10:41:56,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:56,080 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:56,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:56,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:56,118 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733740916101, max timestamp=1733740916114Num uniques 1086; 2024-12-09T10:41:56,118 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733740916101, max timestamp=1733740916114Num uniques 1086; 2024-12-09T10:41:56,118 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=8516, suffix heap size=140912,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-12-09T10:41:56,118 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=8516, suffix heap size=140912,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-12-09T10:41:56,118 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733740916101, max timestamp=1733740916114Num uniques 1086; 2024-12-09T10:41:56,119 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=8516, suffix heap size=140912,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-12-09T10:41:56,171 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1733740916153, max timestamp=1733740916166Num uniques 1045; 2024-12-09T10:41:56,171 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-12-09T10:41:56,171 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1733740916153, max timestamp=1733740916166Num uniques 1045; 2024-12-09T10:41:56,171 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-12-09T10:41:56,171 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1733740916153, max timestamp=1733740916166Num uniques 1045; 2024-12-09T10:41:56,171 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-12-09T10:41:56,218 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1733740916192, max timestamp=1733740916214Num uniques 1054; 2024-12-09T10:41:56,218 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1733740916192, max timestamp=1733740916214Num uniques 1054; 2024-12-09T10:41:56,218 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=9340, suffix heap size=137584,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=204 2024-12-09T10:41:56,218 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=9340, suffix heap size=137584,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=204 2024-12-09T10:41:56,219 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1733740916192, max timestamp=1733740916214Num uniques 1054; 2024-12-09T10:41:56,219 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=9340, suffix heap size=137584,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=204 2024-12-09T10:41:56,233 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-09T10:41:56,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:56,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. as already flushing 2024-12-09T10:41:56,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:56,233 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:56,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:56,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:56,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-09T10:41:56,277 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1733740916258, max timestamp=1733740916272Num uniques 1063; 2024-12-09T10:41:56,277 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1733740916258, max timestamp=1733740916272Num uniques 1063; 2024-12-09T10:41:56,277 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=9748, suffix heap size=138520,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=213 2024-12-09T10:41:56,277 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1733740916258, max timestamp=1733740916272Num uniques 1063; 2024-12-09T10:41:56,277 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=9748, suffix heap size=138520,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=213 2024-12-09T10:41:56,277 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=9748, suffix heap size=138520,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=213 2024-12-09T10:41:56,329 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1733740916312, max timestamp=1733740916324Num uniques 1072; 2024-12-09T10:41:56,329 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1733740916312, max timestamp=1733740916324Num uniques 1072; 2024-12-09T10:41:56,329 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1733740916312, max timestamp=1733740916324Num uniques 1072; 2024-12-09T10:41:56,329 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=10162, suffix heap size=139456,new segment heap size=23328 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=222 2024-12-09T10:41:56,329 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=10162, suffix heap size=139456,new segment heap size=23328 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=222 2024-12-09T10:41:56,329 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=10162, suffix heap size=139456,new segment heap size=23328 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=222 2024-12-09T10:41:56,339 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/C/c803a4d7149546b0a1302fdd47e8790d 2024-12-09T10:41:56,345 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/A/753cc2ad7431459fbc209c1bc6714a95 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/753cc2ad7431459fbc209c1bc6714a95 2024-12-09T10:41:56,351 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/753cc2ad7431459fbc209c1bc6714a95, entries=150, sequenceid=365, filesize=12.0 K 2024-12-09T10:41:56,352 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/B/b55f1eb557544697ab9b49f2641e2c30 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/b55f1eb557544697ab9b49f2641e2c30 2024-12-09T10:41:56,356 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/b55f1eb557544697ab9b49f2641e2c30, entries=150, sequenceid=365, filesize=12.0 K 2024-12-09T10:41:56,356 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/C/c803a4d7149546b0a1302fdd47e8790d as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/c803a4d7149546b0a1302fdd47e8790d 2024-12-09T10:41:56,360 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/c803a4d7149546b0a1302fdd47e8790d, entries=150, sequenceid=365, filesize=12.0 K 2024-12-09T10:41:56,361 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~80.58 KB/82512, currentSize=107.34 KB/109920 for a9bc2b5e1505540f70456fbe90600f90 in 466ms, sequenceid=365, compaction requested=true 2024-12-09T10:41:56,361 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a9bc2b5e1505540f70456fbe90600f90: 2024-12-09T10:41:56,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9bc2b5e1505540f70456fbe90600f90:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T10:41:56,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:41:56,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9bc2b5e1505540f70456fbe90600f90:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T10:41:56,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:41:56,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9bc2b5e1505540f70456fbe90600f90:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T10:41:56,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(8855): Flush requested on a9bc2b5e1505540f70456fbe90600f90 2024-12-09T10:41:56,361 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T10:41:56,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T10:41:56,361 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T10:41:56,361 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a9bc2b5e1505540f70456fbe90600f90 3/3 column families, dataSize=114.05 KB heapSize=319.31 KB 2024-12-09T10:41:56,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9bc2b5e1505540f70456fbe90600f90, store=A 2024-12-09T10:41:56,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T10:41:56,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9bc2b5e1505540f70456fbe90600f90, store=B 2024-12-09T10:41:56,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T10:41:56,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9bc2b5e1505540f70456fbe90600f90, store=C 2024-12-09T10:41:56,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T10:41:56,362 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T10:41:56,362 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T10:41:56,363 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1541): a9bc2b5e1505540f70456fbe90600f90/B is initiating minor compaction (all files) 2024-12-09T10:41:56,363 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1541): a9bc2b5e1505540f70456fbe90600f90/A is initiating minor compaction (all files) 2024-12-09T10:41:56,363 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a9bc2b5e1505540f70456fbe90600f90/B in TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:56,363 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a9bc2b5e1505540f70456fbe90600f90/A in TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:56,363 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/3eca2fe3703d4d6eb24d49f1ff3a2395, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/a89387fe7ef14e278012eb283087a4c0, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/ff9bd9d6e35f45109139d5b08f7e2c38, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/b55f1eb557544697ab9b49f2641e2c30] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp, totalSize=48.8 K 2024-12-09T10:41:56,363 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/093db4477dab446288957c279bb5f1c4, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/9f7e54c866b4411084e1aa97efa573c3, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/320c0829b8f54f869b1023ae2ebebce8, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/753cc2ad7431459fbc209c1bc6714a95] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp, totalSize=48.8 K 2024-12-09T10:41:56,363 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 3eca2fe3703d4d6eb24d49f1ff3a2395, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1733740912206 2024-12-09T10:41:56,364 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 093db4477dab446288957c279bb5f1c4, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1733740912206 2024-12-09T10:41:56,364 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting a89387fe7ef14e278012eb283087a4c0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1733740912329 2024-12-09T10:41:56,364 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9f7e54c866b4411084e1aa97efa573c3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1733740912329 2024-12-09T10:41:56,364 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting ff9bd9d6e35f45109139d5b08f7e2c38, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1733740912979 2024-12-09T10:41:56,365 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 320c0829b8f54f869b1023ae2ebebce8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1733740912979 2024-12-09T10:41:56,365 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting b55f1eb557544697ab9b49f2641e2c30, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1733740915241 2024-12-09T10:41:56,365 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 753cc2ad7431459fbc209c1bc6714a95, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1733740915241 2024-12-09T10:41:56,368 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/A/db8b28b4e3024bbca8982812c5a44c64 is 50, key is test_row_0/A:col10/1733740916345/Put/seqid=0 2024-12-09T10:41:56,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742142_1318 (size=17181) 2024-12-09T10:41:56,378 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a9bc2b5e1505540f70456fbe90600f90#B#compaction#274 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:41:56,378 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/B/1c2972a258eb42f7b215354777568167 is 50, key is test_row_0/B:col10/1733740915247/Put/seqid=0 2024-12-09T10:41:56,379 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a9bc2b5e1505540f70456fbe90600f90#A#compaction#275 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:41:56,380 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/A/41c5e37c4bb440d5a4cfe8e586d5fbb0 is 50, key is test_row_0/A:col10/1733740915247/Put/seqid=0 2024-12-09T10:41:56,386 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-09T10:41:56,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:56,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. as already flushing 2024-12-09T10:41:56,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:56,387 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:56,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:56,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:56,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742143_1319 (size=13187) 2024-12-09T10:41:56,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742144_1320 (size=13187) 2024-12-09T10:41:56,399 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/B/1c2972a258eb42f7b215354777568167 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/1c2972a258eb42f7b215354777568167 2024-12-09T10:41:56,400 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/A/41c5e37c4bb440d5a4cfe8e586d5fbb0 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/41c5e37c4bb440d5a4cfe8e586d5fbb0 2024-12-09T10:41:56,407 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a9bc2b5e1505540f70456fbe90600f90/B of a9bc2b5e1505540f70456fbe90600f90 into 1c2972a258eb42f7b215354777568167(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:41:56,407 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a9bc2b5e1505540f70456fbe90600f90: 2024-12-09T10:41:56,407 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., storeName=a9bc2b5e1505540f70456fbe90600f90/B, priority=12, startTime=1733740916361; duration=0sec 2024-12-09T10:41:56,407 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T10:41:56,407 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9bc2b5e1505540f70456fbe90600f90:B 2024-12-09T10:41:56,407 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T10:41:56,408 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:56,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 234 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50390 deadline: 1733740976401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:56,408 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:56,408 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:56,408 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:56,410 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a9bc2b5e1505540f70456fbe90600f90/A of a9bc2b5e1505540f70456fbe90600f90 into 41c5e37c4bb440d5a4cfe8e586d5fbb0(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:41:56,410 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a9bc2b5e1505540f70456fbe90600f90: 2024-12-09T10:41:56,410 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., storeName=a9bc2b5e1505540f70456fbe90600f90/A, priority=12, startTime=1733740916361; duration=0sec 2024-12-09T10:41:56,410 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T10:41:56,410 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:41:56,410 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9bc2b5e1505540f70456fbe90600f90:A 2024-12-09T10:41:56,410 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1541): a9bc2b5e1505540f70456fbe90600f90/C is initiating minor compaction (all files) 2024-12-09T10:41:56,410 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a9bc2b5e1505540f70456fbe90600f90/C in TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:56,410 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/c15fa0fb2b3f4478b2769fa9501dbe82, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/07242d4f52b749789eb5b02297f6ec71, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/8896a3f96c3046cd86cd3de1ab0e4551, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/c803a4d7149546b0a1302fdd47e8790d] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp, totalSize=48.8 K 2024-12-09T10:41:56,411 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting c15fa0fb2b3f4478b2769fa9501dbe82, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1733740912206 2024-12-09T10:41:56,411 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 07242d4f52b749789eb5b02297f6ec71, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1733740912329 2024-12-09T10:41:56,411 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 8896a3f96c3046cd86cd3de1ab0e4551, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1733740912979 2024-12-09T10:41:56,412 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting c803a4d7149546b0a1302fdd47e8790d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1733740915241 2024-12-09T10:41:56,413 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:56,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 233 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50340 deadline: 1733740976407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:56,414 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:56,414 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:56,414 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:56,429 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a9bc2b5e1505540f70456fbe90600f90#C#compaction#276 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:41:56,430 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/C/5ee8cfd32b8d40d5889927b15da9a30e is 50, key is test_row_0/C:col10/1733740915247/Put/seqid=0 2024-12-09T10:41:56,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742145_1321 (size=13187) 2024-12-09T10:41:56,444 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/C/5ee8cfd32b8d40d5889927b15da9a30e as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/5ee8cfd32b8d40d5889927b15da9a30e 2024-12-09T10:41:56,450 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a9bc2b5e1505540f70456fbe90600f90/C of a9bc2b5e1505540f70456fbe90600f90 into 5ee8cfd32b8d40d5889927b15da9a30e(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:41:56,450 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a9bc2b5e1505540f70456fbe90600f90: 2024-12-09T10:41:56,450 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., storeName=a9bc2b5e1505540f70456fbe90600f90/C, priority=12, startTime=1733740916361; duration=0sec 2024-12-09T10:41:56,451 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:41:56,451 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9bc2b5e1505540f70456fbe90600f90:C 2024-12-09T10:41:56,524 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:56,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 235 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50390 deadline: 1733740976519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:56,525 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:56,525 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:56,525 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:56,525 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:56,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 234 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50340 deadline: 1733740976519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:56,525 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:56,525 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:56,525 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:56,539 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-09T10:41:56,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:56,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. as already flushing 2024-12-09T10:41:56,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:56,540 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:56,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:56,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:56,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-09T10:41:56,571 INFO [Thread-1132 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1132. 2024-12-09T10:41:56,571 DEBUG [Thread-1132 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T10:41:56,571 DEBUG [Thread-1132 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:41:56,571 DEBUG [Thread-1132 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:41:56,571 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T10:41:56,572 INFO [Thread-1128 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1128. 2024-12-09T10:41:56,572 DEBUG [Thread-1128 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T10:41:56,572 DEBUG [Thread-1128 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:41:56,572 DEBUG [Thread-1128 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:41:56,572 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T10:41:56,572 INFO [Thread-1134 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1134. 2024-12-09T10:41:56,572 DEBUG [Thread-1134 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T10:41:56,572 DEBUG [Thread-1134 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:41:56,572 DEBUG [Thread-1134 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:41:56,573 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T10:41:56,574 INFO [Thread-1130 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1130. 2024-12-09T10:41:56,574 DEBUG [Thread-1130 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T10:41:56,574 DEBUG [Thread-1130 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:41:56,574 DEBUG [Thread-1130 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:41:56,574 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T10:41:56,576 INFO [Thread-1126 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1126. 2024-12-09T10:41:56,576 DEBUG [Thread-1126 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T10:41:56,576 DEBUG [Thread-1126 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:41:56,576 DEBUG [Thread-1126 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:41:56,576 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T10:41:56,650 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:56,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50408 deadline: 1733740976650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:56,651 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 7ea43b39fc24,36101,1733740827678 for 'test_row_2' in a9bc2b5e1505540f70456fbe90600f90 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18226 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T10:41:56,651 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:56,651 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:56,651 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:56,692 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-09T10:41:56,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:56,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. as already flushing 2024-12-09T10:41:56,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:56,693 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:56,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:56,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:56,730 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:56,730 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:56,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 235 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50340 deadline: 1733740976730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:56,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 236 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50390 deadline: 1733740976730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:56,730 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:56,730 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:56,730 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:56,730 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:56,730 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:56,730 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:56,759 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:56,759 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:56,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50422 deadline: 1733740976759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:56,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50410 deadline: 1733740976759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:56,760 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 7ea43b39fc24,36101,1733740827678 for 'test_row_0' in a9bc2b5e1505540f70456fbe90600f90 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8232 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T10:41:56,760 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 7ea43b39fc24,36101,1733740827678 for 'test_row_1' in a9bc2b5e1505540f70456fbe90600f90 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8232 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T10:41:56,760 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:56,760 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:56,760 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:56,760 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:56,760 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:56,760 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:56,776 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=518 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/A/db8b28b4e3024bbca8982812c5a44c64 2024-12-09T10:41:56,782 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/B/1edc3089c88048f7afd0074daef8a148 is 50, key is test_row_0/B:col10/1733740916345/Put/seqid=0 2024-12-09T10:41:56,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742146_1322 (size=12301) 2024-12-09T10:41:56,845 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-09T10:41:56,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:56,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. as already flushing 2024-12-09T10:41:56,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:56,846 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:56,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:56,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:56,998 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-09T10:41:56,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:56,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. as already flushing 2024-12-09T10:41:56,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:56,999 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:56,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:56,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:57,040 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:57,040 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:57,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 236 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50340 deadline: 1733740977039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:57,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 237 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50390 deadline: 1733740977039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:57,040 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:57,040 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:57,040 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:57,040 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:57,040 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:57,040 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:57,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-09T10:41:57,150 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-09T10:41:57,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:57,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. as already flushing 2024-12-09T10:41:57,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:57,151 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:57,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:57,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:57,186 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=518 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/B/1edc3089c88048f7afd0074daef8a148 2024-12-09T10:41:57,192 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/C/b2aec2a43eab445d87c7f12d93b06321 is 50, key is test_row_0/C:col10/1733740916345/Put/seqid=0 2024-12-09T10:41:57,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742147_1323 (size=12301) 2024-12-09T10:41:57,303 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-09T10:41:57,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:57,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. as already flushing 2024-12-09T10:41:57,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:57,304 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:57,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:57,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:57,457 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-09T10:41:57,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:57,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. as already flushing 2024-12-09T10:41:57,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:57,457 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:57,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:57,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:57,550 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:57,550 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:41:57,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 237 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50340 deadline: 1733740977549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:57,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 238 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50390 deadline: 1733740977549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:41:57,550 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:57,550 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:57,550 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:57,550 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:41:57,550 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:57,550 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:41:57,596 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=518 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/C/b2aec2a43eab445d87c7f12d93b06321 2024-12-09T10:41:57,600 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/A/db8b28b4e3024bbca8982812c5a44c64 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/db8b28b4e3024bbca8982812c5a44c64 2024-12-09T10:41:57,603 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/db8b28b4e3024bbca8982812c5a44c64, entries=250, sequenceid=518, filesize=16.8 K 2024-12-09T10:41:57,604 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/B/1edc3089c88048f7afd0074daef8a148 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/1edc3089c88048f7afd0074daef8a148 2024-12-09T10:41:57,607 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/1edc3089c88048f7afd0074daef8a148, entries=150, sequenceid=518, filesize=12.0 K 2024-12-09T10:41:57,608 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/C/b2aec2a43eab445d87c7f12d93b06321 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/b2aec2a43eab445d87c7f12d93b06321 2024-12-09T10:41:57,610 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-09T10:41:57,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:57,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. as already flushing 2024-12-09T10:41:57,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:57,610 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:57,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:57,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:41:57,611 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/b2aec2a43eab445d87c7f12d93b06321, entries=150, sequenceid=518, filesize=12.0 K 2024-12-09T10:41:57,612 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~319.27 KB/326928, currentSize=67.09 KB/68700 for a9bc2b5e1505540f70456fbe90600f90 in 1251ms, sequenceid=518, compaction requested=false 2024-12-09T10:41:57,612 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a9bc2b5e1505540f70456fbe90600f90: 2024-12-09T10:41:57,763 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-09T10:41:57,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:57,763 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing a9bc2b5e1505540f70456fbe90600f90 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-09T10:41:57,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9bc2b5e1505540f70456fbe90600f90, store=A 2024-12-09T10:41:57,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:41:57,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9bc2b5e1505540f70456fbe90600f90, store=B 2024-12-09T10:41:57,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:41:57,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9bc2b5e1505540f70456fbe90600f90, store=C 2024-12-09T10:41:57,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:41:57,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/A/5c8473272e3548edae8b67634c55fb17 is 50, key is test_row_0/A:col10/1733740916404/Put/seqid=0 2024-12-09T10:41:57,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742148_1324 (size=12301) 2024-12-09T10:41:58,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-09T10:41:58,171 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=534 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/A/5c8473272e3548edae8b67634c55fb17 2024-12-09T10:41:58,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/B/81718056a7c34a7e822aec04171bc59b is 50, key is test_row_0/B:col10/1733740916404/Put/seqid=0 2024-12-09T10:41:58,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742149_1325 (size=12301) 2024-12-09T10:41:58,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(8855): Flush requested on a9bc2b5e1505540f70456fbe90600f90 2024-12-09T10:41:58,561 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. as already flushing 2024-12-09T10:41:58,561 INFO [Thread-1117 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1117. 2024-12-09T10:41:58,561 DEBUG [Thread-1117 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T10:41:58,561 DEBUG [Thread-1117 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:41:58,561 DEBUG [Thread-1117 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:41:58,561 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T10:41:58,570 INFO [Thread-1121 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1121. 2024-12-09T10:41:58,570 DEBUG [Thread-1121 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T10:41:58,570 DEBUG [Thread-1121 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:41:58,570 DEBUG [Thread-1121 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:41:58,570 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T10:41:58,581 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=534 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/B/81718056a7c34a7e822aec04171bc59b 2024-12-09T10:41:58,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/C/458e1f83a7ef47bcab97b21ca9b31fa4 is 50, key is test_row_0/C:col10/1733740916404/Put/seqid=0 2024-12-09T10:41:58,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742150_1326 (size=12301) 2024-12-09T10:41:58,991 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=534 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/C/458e1f83a7ef47bcab97b21ca9b31fa4 2024-12-09T10:41:58,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/A/5c8473272e3548edae8b67634c55fb17 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/5c8473272e3548edae8b67634c55fb17 2024-12-09T10:41:58,999 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/5c8473272e3548edae8b67634c55fb17, entries=150, sequenceid=534, filesize=12.0 K 2024-12-09T10:41:59,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/B/81718056a7c34a7e822aec04171bc59b as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/81718056a7c34a7e822aec04171bc59b 2024-12-09T10:41:59,003 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/81718056a7c34a7e822aec04171bc59b, entries=150, sequenceid=534, filesize=12.0 K 2024-12-09T10:41:59,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/C/458e1f83a7ef47bcab97b21ca9b31fa4 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/458e1f83a7ef47bcab97b21ca9b31fa4 2024-12-09T10:41:59,006 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/458e1f83a7ef47bcab97b21ca9b31fa4, entries=150, sequenceid=534, filesize=12.0 K 2024-12-09T10:41:59,007 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=13.42 KB/13740 for a9bc2b5e1505540f70456fbe90600f90 in 1244ms, sequenceid=534, compaction requested=true 2024-12-09T10:41:59,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for a9bc2b5e1505540f70456fbe90600f90: 2024-12-09T10:41:59,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:41:59,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-12-09T10:41:59,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-12-09T10:41:59,010 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-12-09T10:41:59,010 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0830 sec 2024-12-09T10:41:59,012 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 3.0870 sec 2024-12-09T10:42:00,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-09T10:42:00,090 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T10:42:06,671 INFO [Thread-1115 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1115. 2024-12-09T10:42:06,671 DEBUG [Thread-1115 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T10:42:06,671 DEBUG [Thread-1115 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:42:06,671 DEBUG [Thread-1115 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:42:06,671 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T10:42:06,811 INFO [Thread-1123 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1123. 2024-12-09T10:42:06,811 DEBUG [Thread-1123 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T10:42:06,811 DEBUG [Thread-1123 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:42:06,811 DEBUG [Thread-1123 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:42:06,811 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T10:42:06,851 INFO [Thread-1119 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1119. 2024-12-09T10:42:06,851 DEBUG [Thread-1119 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T10:42:06,851 DEBUG [Thread-1119 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:42:06,851 DEBUG [Thread-1119 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:42:06,851 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T10:42:06,852 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-09T10:42:06,852 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 13 2024-12-09T10:42:06,852 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 175 2024-12-09T10:42:06,852 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 49 2024-12-09T10:42:06,852 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 174 2024-12-09T10:42:06,852 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 40 2024-12-09T10:42:06,852 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-09T10:42:06,852 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3304 2024-12-09T10:42:06,852 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3015 2024-12-09T10:42:06,852 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3190 2024-12-09T10:42:06,852 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3344 2024-12-09T10:42:06,852 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3013 2024-12-09T10:42:06,852 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-09T10:42:06,852 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-09T10:42:06,852 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T10:42:06,852 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:42:06,852 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:42:06,852 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T10:42:06,853 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-12-09T10:42:06,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-09T10:42:06,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-09T10:42:06,856 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733740926856"}]},"ts":"1733740926856"} 2024-12-09T10:42:06,857 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-09T10:42:06,857 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-09T10:42:06,858 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-09T10:42:06,859 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a9bc2b5e1505540f70456fbe90600f90, UNASSIGN}] 2024-12-09T10:42:06,859 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a9bc2b5e1505540f70456fbe90600f90, UNASSIGN 2024-12-09T10:42:06,860 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=82 updating hbase:meta row=a9bc2b5e1505540f70456fbe90600f90, regionState=CLOSING, regionLocation=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:06,861 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a9bc2b5e1505540f70456fbe90600f90, UNASSIGN because future has completed 2024-12-09T10:42:06,861 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-09T10:42:06,861 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; CloseRegionProcedure a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678}] 2024-12-09T10:42:06,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-09T10:42:07,013 INFO [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] handler.UnassignRegionHandler(122): Close a9bc2b5e1505540f70456fbe90600f90 2024-12-09T10:42:07,013 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-09T10:42:07,014 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1722): Closing a9bc2b5e1505540f70456fbe90600f90, disabling compactions & flushes 2024-12-09T10:42:07,014 INFO [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:42:07,014 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:42:07,014 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. after waiting 0 ms 2024-12-09T10:42:07,014 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:42:07,014 INFO [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(2902): Flushing a9bc2b5e1505540f70456fbe90600f90 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-09T10:42:07,014 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9bc2b5e1505540f70456fbe90600f90, store=A 2024-12-09T10:42:07,014 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:42:07,014 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9bc2b5e1505540f70456fbe90600f90, store=B 2024-12-09T10:42:07,014 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:42:07,014 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9bc2b5e1505540f70456fbe90600f90, store=C 2024-12-09T10:42:07,014 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:42:07,018 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/A/a55b97578dbb43b78a8c097fc90ea536 is 50, key is test_row_0/A:col10/1733740926850/Put/seqid=0 2024-12-09T10:42:07,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742151_1327 (size=12301) 2024-12-09T10:42:07,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-09T10:42:07,422 INFO [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=542 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/A/a55b97578dbb43b78a8c097fc90ea536 2024-12-09T10:42:07,428 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/B/0789e6174c8e4af2ad22618b0c74e30e is 50, key is test_row_0/B:col10/1733740926850/Put/seqid=0 2024-12-09T10:42:07,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742152_1328 (size=12301) 2024-12-09T10:42:07,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-09T10:42:07,832 INFO [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=542 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/B/0789e6174c8e4af2ad22618b0c74e30e 2024-12-09T10:42:07,839 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/C/db9fd0edab284f39b30b477c647271c0 is 50, key is test_row_0/C:col10/1733740926850/Put/seqid=0 2024-12-09T10:42:07,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742153_1329 (size=12301) 2024-12-09T10:42:07,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-09T10:42:08,243 INFO [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=542 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/C/db9fd0edab284f39b30b477c647271c0 2024-12-09T10:42:08,247 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/A/a55b97578dbb43b78a8c097fc90ea536 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/a55b97578dbb43b78a8c097fc90ea536 2024-12-09T10:42:08,251 INFO [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/a55b97578dbb43b78a8c097fc90ea536, entries=150, sequenceid=542, filesize=12.0 K 2024-12-09T10:42:08,251 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/B/0789e6174c8e4af2ad22618b0c74e30e as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/0789e6174c8e4af2ad22618b0c74e30e 2024-12-09T10:42:08,254 INFO [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/0789e6174c8e4af2ad22618b0c74e30e, entries=150, sequenceid=542, filesize=12.0 K 2024-12-09T10:42:08,255 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/.tmp/C/db9fd0edab284f39b30b477c647271c0 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/db9fd0edab284f39b30b477c647271c0 2024-12-09T10:42:08,258 INFO [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/db9fd0edab284f39b30b477c647271c0, entries=150, sequenceid=542, filesize=12.0 K 2024-12-09T10:42:08,259 INFO [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for a9bc2b5e1505540f70456fbe90600f90 in 1245ms, sequenceid=542, compaction requested=true 2024-12-09T10:42:08,259 DEBUG [StoreCloser-TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/c151e1a164ac49ab93c3ee4a6e0dad59, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/83d14cd85d61439c8ac3df3995caa7d2, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/7e065977abbf4c5aa2eb93592b62d999, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/41528c70c5c4471084f0037919b20b35, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/b8e20005785443c58e347a18f9f8d8bf, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/c23fa272d24c4008abced08ff5154d9c, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/091e62da52bc4eec8dfea0254396a1a0, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/5bd267c6b30e49688ce75cb65465303f, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/401b23b8cc244183a8a21482bb6e12cd, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/4ee57521787443d789dce1c957e97870, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/298a0d7f7bbb4c44a87a2700bb845f41, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/276a443b02574b11a7bc2d1c7cf15c66, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/92d349a3ea3e49e2bec79038736cc38f, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/77d139004e344877b1d3acabeb82cf9a, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/918af91bd0dc43749111b7d0a4c56db8, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/928c531e88b4442089a3a56c3afa7565, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/89076f62549c44eaab27987207b7307b, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/1f587f228c5d4b318ed3eed223fe3249, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/fe5cf388ae6f4d9ab5c2a40a6f12452d, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/fefc428ace66468f8584f0f2b5e64fd7, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/29f767df228e4b0998b8e64c76fc7481, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/91291059081a40259f14082260b0e316, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/093db4477dab446288957c279bb5f1c4, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/9f7e54c866b4411084e1aa97efa573c3, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/320c0829b8f54f869b1023ae2ebebce8, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/753cc2ad7431459fbc209c1bc6714a95] to archive 2024-12-09T10:42:08,260 DEBUG [StoreCloser-TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T10:42:08,262 DEBUG [StoreCloser-TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/c151e1a164ac49ab93c3ee4a6e0dad59 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/c151e1a164ac49ab93c3ee4a6e0dad59 2024-12-09T10:42:08,263 DEBUG [StoreCloser-TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/83d14cd85d61439c8ac3df3995caa7d2 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/83d14cd85d61439c8ac3df3995caa7d2 2024-12-09T10:42:08,264 DEBUG [StoreCloser-TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/7e065977abbf4c5aa2eb93592b62d999 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/7e065977abbf4c5aa2eb93592b62d999 2024-12-09T10:42:08,265 DEBUG [StoreCloser-TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/41528c70c5c4471084f0037919b20b35 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/41528c70c5c4471084f0037919b20b35 2024-12-09T10:42:08,266 DEBUG [StoreCloser-TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/b8e20005785443c58e347a18f9f8d8bf to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/b8e20005785443c58e347a18f9f8d8bf 2024-12-09T10:42:08,266 DEBUG [StoreCloser-TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/c23fa272d24c4008abced08ff5154d9c to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/c23fa272d24c4008abced08ff5154d9c 2024-12-09T10:42:08,267 DEBUG [StoreCloser-TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/091e62da52bc4eec8dfea0254396a1a0 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/091e62da52bc4eec8dfea0254396a1a0 2024-12-09T10:42:08,268 DEBUG [StoreCloser-TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/5bd267c6b30e49688ce75cb65465303f to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/5bd267c6b30e49688ce75cb65465303f 2024-12-09T10:42:08,269 DEBUG [StoreCloser-TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/401b23b8cc244183a8a21482bb6e12cd to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/401b23b8cc244183a8a21482bb6e12cd 2024-12-09T10:42:08,270 DEBUG [StoreCloser-TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/4ee57521787443d789dce1c957e97870 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/4ee57521787443d789dce1c957e97870 2024-12-09T10:42:08,271 DEBUG [StoreCloser-TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/298a0d7f7bbb4c44a87a2700bb845f41 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/298a0d7f7bbb4c44a87a2700bb845f41 2024-12-09T10:42:08,272 DEBUG [StoreCloser-TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/276a443b02574b11a7bc2d1c7cf15c66 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/276a443b02574b11a7bc2d1c7cf15c66 2024-12-09T10:42:08,273 DEBUG [StoreCloser-TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/92d349a3ea3e49e2bec79038736cc38f to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/92d349a3ea3e49e2bec79038736cc38f 2024-12-09T10:42:08,274 DEBUG [StoreCloser-TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/77d139004e344877b1d3acabeb82cf9a to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/77d139004e344877b1d3acabeb82cf9a 2024-12-09T10:42:08,274 DEBUG [StoreCloser-TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/918af91bd0dc43749111b7d0a4c56db8 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/918af91bd0dc43749111b7d0a4c56db8 2024-12-09T10:42:08,275 DEBUG [StoreCloser-TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/928c531e88b4442089a3a56c3afa7565 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/928c531e88b4442089a3a56c3afa7565 2024-12-09T10:42:08,276 DEBUG [StoreCloser-TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/89076f62549c44eaab27987207b7307b to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/89076f62549c44eaab27987207b7307b 2024-12-09T10:42:08,277 DEBUG [StoreCloser-TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/1f587f228c5d4b318ed3eed223fe3249 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/1f587f228c5d4b318ed3eed223fe3249 2024-12-09T10:42:08,278 DEBUG [StoreCloser-TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/fe5cf388ae6f4d9ab5c2a40a6f12452d to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/fe5cf388ae6f4d9ab5c2a40a6f12452d 2024-12-09T10:42:08,279 DEBUG [StoreCloser-TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/fefc428ace66468f8584f0f2b5e64fd7 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/fefc428ace66468f8584f0f2b5e64fd7 2024-12-09T10:42:08,280 DEBUG [StoreCloser-TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/29f767df228e4b0998b8e64c76fc7481 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/29f767df228e4b0998b8e64c76fc7481 2024-12-09T10:42:08,280 DEBUG [StoreCloser-TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/91291059081a40259f14082260b0e316 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/91291059081a40259f14082260b0e316 2024-12-09T10:42:08,281 DEBUG [StoreCloser-TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/093db4477dab446288957c279bb5f1c4 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/093db4477dab446288957c279bb5f1c4 2024-12-09T10:42:08,282 DEBUG [StoreCloser-TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/9f7e54c866b4411084e1aa97efa573c3 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/9f7e54c866b4411084e1aa97efa573c3 2024-12-09T10:42:08,283 DEBUG [StoreCloser-TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/320c0829b8f54f869b1023ae2ebebce8 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/320c0829b8f54f869b1023ae2ebebce8 2024-12-09T10:42:08,284 DEBUG [StoreCloser-TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/753cc2ad7431459fbc209c1bc6714a95 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/753cc2ad7431459fbc209c1bc6714a95 2024-12-09T10:42:08,285 DEBUG [StoreCloser-TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/0534f3f0656840f494cf63787da7e735, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/42a64d85a8c94b1a95a1d75281a0c2ad, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/7ffe5f63c06846bea7fb035e9a61eaf7, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/39a8862ba4e742c182751de99eb3dcd2, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/0d23e59a4089427e9895a130af9c4713, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/59efd1b2f91240358a14c3756581b2a3, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/410ea7949ac64800afefaed9508f5acb, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/60c47024b4174321ad27b2a7623e8e19, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/e6e8752c2bd64658abd3d9599c6c8931, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/515e28dc83924127bb5e582d354544ef, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/ae1722c9079a43f6bb386c6d779398ab, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/5118f60a7f854379a04dd8f38ba1044a, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/23d19da71fc34a67bb3e5357e6ccf5ae, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/261a9256666244368cee662d2c8ba81b, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/8ed89976b87d4de0b6d13592a65a621b, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/ef034b024da04d83964895652bdad55d, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/b061c9252d91485a958680a4aec5f63a, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/62ad861ea4234cf983211efbc85edf9c, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/1a2d950d62634c5a9a0b75c24c59ba09, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/93900fb0c76f46ab8a704c71ada382ae, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/321fba8eefe64b88be59a433c76d8f02, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/3eca2fe3703d4d6eb24d49f1ff3a2395, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/8a408e3b15d1482686ddedea512cc68d, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/a89387fe7ef14e278012eb283087a4c0, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/ff9bd9d6e35f45109139d5b08f7e2c38, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/b55f1eb557544697ab9b49f2641e2c30] to archive 2024-12-09T10:42:08,286 DEBUG [StoreCloser-TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T10:42:08,287 DEBUG [StoreCloser-TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/0534f3f0656840f494cf63787da7e735 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/0534f3f0656840f494cf63787da7e735 2024-12-09T10:42:08,288 DEBUG [StoreCloser-TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/42a64d85a8c94b1a95a1d75281a0c2ad to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/42a64d85a8c94b1a95a1d75281a0c2ad 2024-12-09T10:42:08,289 DEBUG [StoreCloser-TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/7ffe5f63c06846bea7fb035e9a61eaf7 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/7ffe5f63c06846bea7fb035e9a61eaf7 2024-12-09T10:42:08,290 DEBUG [StoreCloser-TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/39a8862ba4e742c182751de99eb3dcd2 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/39a8862ba4e742c182751de99eb3dcd2 2024-12-09T10:42:08,291 DEBUG [StoreCloser-TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/0d23e59a4089427e9895a130af9c4713 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/0d23e59a4089427e9895a130af9c4713 2024-12-09T10:42:08,292 DEBUG [StoreCloser-TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/59efd1b2f91240358a14c3756581b2a3 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/59efd1b2f91240358a14c3756581b2a3 2024-12-09T10:42:08,293 DEBUG [StoreCloser-TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/410ea7949ac64800afefaed9508f5acb to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/410ea7949ac64800afefaed9508f5acb 2024-12-09T10:42:08,294 DEBUG [StoreCloser-TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/60c47024b4174321ad27b2a7623e8e19 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/60c47024b4174321ad27b2a7623e8e19 2024-12-09T10:42:08,294 DEBUG [StoreCloser-TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/e6e8752c2bd64658abd3d9599c6c8931 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/e6e8752c2bd64658abd3d9599c6c8931 2024-12-09T10:42:08,295 DEBUG [StoreCloser-TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/515e28dc83924127bb5e582d354544ef to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/515e28dc83924127bb5e582d354544ef 2024-12-09T10:42:08,296 DEBUG [StoreCloser-TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/ae1722c9079a43f6bb386c6d779398ab to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/ae1722c9079a43f6bb386c6d779398ab 2024-12-09T10:42:08,297 DEBUG [StoreCloser-TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/5118f60a7f854379a04dd8f38ba1044a to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/5118f60a7f854379a04dd8f38ba1044a 2024-12-09T10:42:08,298 DEBUG [StoreCloser-TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/23d19da71fc34a67bb3e5357e6ccf5ae to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/23d19da71fc34a67bb3e5357e6ccf5ae 2024-12-09T10:42:08,299 DEBUG [StoreCloser-TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/261a9256666244368cee662d2c8ba81b to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/261a9256666244368cee662d2c8ba81b 2024-12-09T10:42:08,300 DEBUG [StoreCloser-TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/8ed89976b87d4de0b6d13592a65a621b to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/8ed89976b87d4de0b6d13592a65a621b 2024-12-09T10:42:08,301 DEBUG [StoreCloser-TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/ef034b024da04d83964895652bdad55d to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/ef034b024da04d83964895652bdad55d 2024-12-09T10:42:08,302 DEBUG [StoreCloser-TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/b061c9252d91485a958680a4aec5f63a to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/b061c9252d91485a958680a4aec5f63a 2024-12-09T10:42:08,303 DEBUG [StoreCloser-TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/62ad861ea4234cf983211efbc85edf9c to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/62ad861ea4234cf983211efbc85edf9c 2024-12-09T10:42:08,303 DEBUG [StoreCloser-TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/1a2d950d62634c5a9a0b75c24c59ba09 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/1a2d950d62634c5a9a0b75c24c59ba09 2024-12-09T10:42:08,304 DEBUG [StoreCloser-TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/93900fb0c76f46ab8a704c71ada382ae to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/93900fb0c76f46ab8a704c71ada382ae 2024-12-09T10:42:08,305 DEBUG [StoreCloser-TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/321fba8eefe64b88be59a433c76d8f02 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/321fba8eefe64b88be59a433c76d8f02 2024-12-09T10:42:08,306 DEBUG [StoreCloser-TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/3eca2fe3703d4d6eb24d49f1ff3a2395 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/3eca2fe3703d4d6eb24d49f1ff3a2395 2024-12-09T10:42:08,307 DEBUG [StoreCloser-TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/8a408e3b15d1482686ddedea512cc68d to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/8a408e3b15d1482686ddedea512cc68d 2024-12-09T10:42:08,308 DEBUG [StoreCloser-TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/a89387fe7ef14e278012eb283087a4c0 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/a89387fe7ef14e278012eb283087a4c0 2024-12-09T10:42:08,310 DEBUG [StoreCloser-TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/ff9bd9d6e35f45109139d5b08f7e2c38 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/ff9bd9d6e35f45109139d5b08f7e2c38 2024-12-09T10:42:08,311 DEBUG [StoreCloser-TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/b55f1eb557544697ab9b49f2641e2c30 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/b55f1eb557544697ab9b49f2641e2c30 2024-12-09T10:42:08,312 DEBUG [StoreCloser-TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/0332bb2996bb402d8e2e02dd6da0bef0, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/dc781da867f4487ca634b88ddadff9dd, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/9d2489362d24431ba9bd18c40d9b2f8d, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/57ba7a2a58c54352b1eed8458309ab5e, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/9d7dd64163954372b82a148680d83fca, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/c36e92c81e094bdfa7588cf23c25a8ba, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/79c84d61d2e94d94ad643bdea8de05ce, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/10b4d8ff98f449b89b3eebdaf9f62c69, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/5218bc8f9e87430f9c9d7323b3eff0e9, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/a727e020dd764dc09c393485c0908644, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/141e2502e0584f848ac0003925dc83b8, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/9e88f495ee964ef4acf875676dfc1818, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/6cb94aee83fc4beb9c55273f274a82b2, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/182b81033c434657ae315668105bfe40, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/16e767b674e24c1eacef216483c0e4cc, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/521d3394712445018a340ba1021aebf2, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/183c50b64f214704b7b923f9d14cbc4f, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/fa1cc8a0acb4472b8c6b3d81cc4dc647, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/e8fa9f75e7864d0b827cdc9d93bdb516, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/ecbe256b1a3c46afa9e34e78dd8b310d, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/3a959d9babc44ee7977ecd144ce50d32, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/c15fa0fb2b3f4478b2769fa9501dbe82, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/796fadd36d244fa5ab03cf8da7e0a43c, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/07242d4f52b749789eb5b02297f6ec71, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/8896a3f96c3046cd86cd3de1ab0e4551, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/c803a4d7149546b0a1302fdd47e8790d] to archive 2024-12-09T10:42:08,313 DEBUG [StoreCloser-TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T10:42:08,314 DEBUG [StoreCloser-TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/0332bb2996bb402d8e2e02dd6da0bef0 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/0332bb2996bb402d8e2e02dd6da0bef0 2024-12-09T10:42:08,316 DEBUG [StoreCloser-TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/dc781da867f4487ca634b88ddadff9dd to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/dc781da867f4487ca634b88ddadff9dd 2024-12-09T10:42:08,317 DEBUG [StoreCloser-TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/9d2489362d24431ba9bd18c40d9b2f8d to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/9d2489362d24431ba9bd18c40d9b2f8d 2024-12-09T10:42:08,318 DEBUG [StoreCloser-TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/57ba7a2a58c54352b1eed8458309ab5e to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/57ba7a2a58c54352b1eed8458309ab5e 2024-12-09T10:42:08,319 DEBUG [StoreCloser-TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/9d7dd64163954372b82a148680d83fca to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/9d7dd64163954372b82a148680d83fca 2024-12-09T10:42:08,321 DEBUG [StoreCloser-TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/c36e92c81e094bdfa7588cf23c25a8ba to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/c36e92c81e094bdfa7588cf23c25a8ba 2024-12-09T10:42:08,322 DEBUG [StoreCloser-TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/79c84d61d2e94d94ad643bdea8de05ce to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/79c84d61d2e94d94ad643bdea8de05ce 2024-12-09T10:42:08,323 DEBUG [StoreCloser-TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/10b4d8ff98f449b89b3eebdaf9f62c69 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/10b4d8ff98f449b89b3eebdaf9f62c69 2024-12-09T10:42:08,324 DEBUG [StoreCloser-TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/5218bc8f9e87430f9c9d7323b3eff0e9 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/5218bc8f9e87430f9c9d7323b3eff0e9 2024-12-09T10:42:08,324 DEBUG [StoreCloser-TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/a727e020dd764dc09c393485c0908644 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/a727e020dd764dc09c393485c0908644 2024-12-09T10:42:08,325 DEBUG [StoreCloser-TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/141e2502e0584f848ac0003925dc83b8 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/141e2502e0584f848ac0003925dc83b8 2024-12-09T10:42:08,326 DEBUG [StoreCloser-TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/9e88f495ee964ef4acf875676dfc1818 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/9e88f495ee964ef4acf875676dfc1818 2024-12-09T10:42:08,327 DEBUG [StoreCloser-TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/6cb94aee83fc4beb9c55273f274a82b2 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/6cb94aee83fc4beb9c55273f274a82b2 2024-12-09T10:42:08,328 DEBUG [StoreCloser-TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/182b81033c434657ae315668105bfe40 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/182b81033c434657ae315668105bfe40 2024-12-09T10:42:08,328 DEBUG [StoreCloser-TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/16e767b674e24c1eacef216483c0e4cc to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/16e767b674e24c1eacef216483c0e4cc 2024-12-09T10:42:08,329 DEBUG [StoreCloser-TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/521d3394712445018a340ba1021aebf2 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/521d3394712445018a340ba1021aebf2 2024-12-09T10:42:08,330 DEBUG [StoreCloser-TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/183c50b64f214704b7b923f9d14cbc4f to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/183c50b64f214704b7b923f9d14cbc4f 2024-12-09T10:42:08,331 DEBUG [StoreCloser-TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/fa1cc8a0acb4472b8c6b3d81cc4dc647 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/fa1cc8a0acb4472b8c6b3d81cc4dc647 2024-12-09T10:42:08,332 DEBUG [StoreCloser-TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/e8fa9f75e7864d0b827cdc9d93bdb516 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/e8fa9f75e7864d0b827cdc9d93bdb516 2024-12-09T10:42:08,332 DEBUG [StoreCloser-TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/ecbe256b1a3c46afa9e34e78dd8b310d to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/ecbe256b1a3c46afa9e34e78dd8b310d 2024-12-09T10:42:08,333 DEBUG [StoreCloser-TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/3a959d9babc44ee7977ecd144ce50d32 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/3a959d9babc44ee7977ecd144ce50d32 2024-12-09T10:42:08,334 DEBUG [StoreCloser-TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/c15fa0fb2b3f4478b2769fa9501dbe82 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/c15fa0fb2b3f4478b2769fa9501dbe82 2024-12-09T10:42:08,335 DEBUG [StoreCloser-TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/796fadd36d244fa5ab03cf8da7e0a43c to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/796fadd36d244fa5ab03cf8da7e0a43c 2024-12-09T10:42:08,336 DEBUG [StoreCloser-TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/07242d4f52b749789eb5b02297f6ec71 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/07242d4f52b749789eb5b02297f6ec71 2024-12-09T10:42:08,337 DEBUG [StoreCloser-TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/8896a3f96c3046cd86cd3de1ab0e4551 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/8896a3f96c3046cd86cd3de1ab0e4551 2024-12-09T10:42:08,337 DEBUG [StoreCloser-TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/c803a4d7149546b0a1302fdd47e8790d to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/c803a4d7149546b0a1302fdd47e8790d 2024-12-09T10:42:08,342 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/recovered.edits/545.seqid, newMaxSeqId=545, maxSeqId=1 2024-12-09T10:42:08,342 INFO [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. 2024-12-09T10:42:08,342 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1676): Region close journal for a9bc2b5e1505540f70456fbe90600f90: Waiting for close lock at 1733740927014Running coprocessor pre-close hooks at 1733740927014Disabling compacts and flushes for region at 1733740927014Disabling writes for close at 1733740927014Obtaining lock to block concurrent updates at 1733740927014Preparing flush snapshotting stores in a9bc2b5e1505540f70456fbe90600f90 at 1733740927014Finished memstore snapshotting TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1733740927014Flushing stores of TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90. at 1733740927015 (+1 ms)Flushing a9bc2b5e1505540f70456fbe90600f90/A: creating writer at 1733740927015Flushing a9bc2b5e1505540f70456fbe90600f90/A: appending metadata at 1733740927017 (+2 ms)Flushing a9bc2b5e1505540f70456fbe90600f90/A: closing flushed file at 1733740927017Flushing a9bc2b5e1505540f70456fbe90600f90/B: creating writer at 1733740927426 (+409 ms)Flushing a9bc2b5e1505540f70456fbe90600f90/B: appending metadata at 1733740927428 (+2 ms)Flushing a9bc2b5e1505540f70456fbe90600f90/B: closing flushed file at 1733740927428Flushing a9bc2b5e1505540f70456fbe90600f90/C: creating writer at 1733740927836 (+408 ms)Flushing a9bc2b5e1505540f70456fbe90600f90/C: appending metadata at 1733740927838 (+2 ms)Flushing a9bc2b5e1505540f70456fbe90600f90/C: closing flushed file at 1733740927838Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@f5d21f3: reopening flushed file at 1733740928247 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@74b24f5f: reopening flushed file at 1733740928251 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4b2886c0: reopening flushed file at 1733740928255 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for a9bc2b5e1505540f70456fbe90600f90 in 1245ms, sequenceid=542, compaction requested=true at 1733740928259 (+4 ms)Writing region close event to WAL at 1733740928339 (+80 ms)Running coprocessor post-close hooks at 1733740928342 (+3 ms)Closed at 1733740928342 2024-12-09T10:42:08,344 INFO [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] handler.UnassignRegionHandler(157): Closed a9bc2b5e1505540f70456fbe90600f90 2024-12-09T10:42:08,345 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=82 updating hbase:meta row=a9bc2b5e1505540f70456fbe90600f90, regionState=CLOSED 2024-12-09T10:42:08,346 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=83, ppid=82, state=RUNNABLE, hasLock=false; CloseRegionProcedure a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 because future has completed 2024-12-09T10:42:08,348 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-12-09T10:42:08,348 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; CloseRegionProcedure a9bc2b5e1505540f70456fbe90600f90, server=7ea43b39fc24,36101,1733740827678 in 1.4860 sec 2024-12-09T10:42:08,349 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=82, resume processing ppid=81 2024-12-09T10:42:08,349 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, ppid=81, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a9bc2b5e1505540f70456fbe90600f90, UNASSIGN in 1.4890 sec 2024-12-09T10:42:08,352 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-12-09T10:42:08,352 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.4930 sec 2024-12-09T10:42:08,353 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733740928352"}]},"ts":"1733740928352"} 2024-12-09T10:42:08,354 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-09T10:42:08,354 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-09T10:42:08,355 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5010 sec 2024-12-09T10:42:09,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-09T10:42:09,000 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-09T10:42:09,001 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-12-09T10:42:09,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T10:42:09,002 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=84, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T10:42:09,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-12-09T10:42:09,003 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=84, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T10:42:09,006 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90 2024-12-09T10:42:09,008 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A, FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B, FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C, FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/recovered.edits] 2024-12-09T10:42:09,010 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/41c5e37c4bb440d5a4cfe8e586d5fbb0 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/41c5e37c4bb440d5a4cfe8e586d5fbb0 2024-12-09T10:42:09,011 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/5c8473272e3548edae8b67634c55fb17 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/5c8473272e3548edae8b67634c55fb17 2024-12-09T10:42:09,012 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/a55b97578dbb43b78a8c097fc90ea536 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/a55b97578dbb43b78a8c097fc90ea536 2024-12-09T10:42:09,013 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/db8b28b4e3024bbca8982812c5a44c64 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/A/db8b28b4e3024bbca8982812c5a44c64 2024-12-09T10:42:09,015 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/0789e6174c8e4af2ad22618b0c74e30e to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/0789e6174c8e4af2ad22618b0c74e30e 2024-12-09T10:42:09,016 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/1c2972a258eb42f7b215354777568167 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/1c2972a258eb42f7b215354777568167 2024-12-09T10:42:09,017 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/1edc3089c88048f7afd0074daef8a148 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/1edc3089c88048f7afd0074daef8a148 2024-12-09T10:42:09,018 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/81718056a7c34a7e822aec04171bc59b to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/B/81718056a7c34a7e822aec04171bc59b 2024-12-09T10:42:09,020 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/458e1f83a7ef47bcab97b21ca9b31fa4 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/458e1f83a7ef47bcab97b21ca9b31fa4 2024-12-09T10:42:09,021 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/5ee8cfd32b8d40d5889927b15da9a30e to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/5ee8cfd32b8d40d5889927b15da9a30e 2024-12-09T10:42:09,022 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/b2aec2a43eab445d87c7f12d93b06321 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/b2aec2a43eab445d87c7f12d93b06321 2024-12-09T10:42:09,023 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/db9fd0edab284f39b30b477c647271c0 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/C/db9fd0edab284f39b30b477c647271c0 2024-12-09T10:42:09,025 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/recovered.edits/545.seqid to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90/recovered.edits/545.seqid 2024-12-09T10:42:09,025 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9bc2b5e1505540f70456fbe90600f90 2024-12-09T10:42:09,025 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-09T10:42:09,027 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=84, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T10:42:09,030 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-09T10:42:09,032 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-09T10:42:09,033 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=84, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T10:42:09,033 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-09T10:42:09,033 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733740929033"}]},"ts":"9223372036854775807"} 2024-12-09T10:42:09,035 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-09T10:42:09,035 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => a9bc2b5e1505540f70456fbe90600f90, NAME => 'TestAcidGuarantees,,1733740894380.a9bc2b5e1505540f70456fbe90600f90.', STARTKEY => '', ENDKEY => ''}] 2024-12-09T10:42:09,035 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-09T10:42:09,035 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733740929035"}]},"ts":"9223372036854775807"} 2024-12-09T10:42:09,037 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-09T10:42:09,037 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=84, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T10:42:09,038 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 36 msec 2024-12-09T10:42:09,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-12-09T10:42:09,110 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-09T10:42:09,113 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-09T10:42:09,120 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=246 (was 248), OpenFileDescriptor=445 (was 451), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=328 (was 418), ProcessCount=11 (was 11), AvailableMemoryMB=7894 (was 8277) 2024-12-09T10:42:09,129 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=246, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=328, ProcessCount=11, AvailableMemoryMB=7894 2024-12-09T10:42:09,131 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-09T10:42:09,131 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T10:42:09,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] procedure2.ProcedureExecutor(1139): Stored pid=85, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-09T10:42:09,133 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-09T10:42:09,133 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:09,133 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 85 2024-12-09T10:42:09,134 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-09T10:42:09,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-12-09T10:42:09,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742154_1330 (size=1061) 2024-12-09T10:42:09,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-12-09T10:42:09,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-12-09T10:42:09,541 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ec3eeaafd032b9a023fe519c0fcbb0c5, NAME => 'TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160 2024-12-09T10:42:09,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742155_1331 (size=53) 2024-12-09T10:42:09,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-12-09T10:42:09,946 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T10:42:09,946 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing ec3eeaafd032b9a023fe519c0fcbb0c5, disabling compactions & flushes 2024-12-09T10:42:09,946 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. 2024-12-09T10:42:09,946 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. 2024-12-09T10:42:09,946 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. after waiting 0 ms 2024-12-09T10:42:09,946 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. 2024-12-09T10:42:09,946 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. 2024-12-09T10:42:09,946 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for ec3eeaafd032b9a023fe519c0fcbb0c5: Waiting for close lock at 1733740929946Disabling compacts and flushes for region at 1733740929946Disabling writes for close at 1733740929946Writing region close event to WAL at 1733740929946Closed at 1733740929946 2024-12-09T10:42:09,947 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-09T10:42:09,948 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733740929947"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733740929947"}]},"ts":"1733740929947"} 2024-12-09T10:42:09,949 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-09T10:42:09,950 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-09T10:42:09,950 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733740929950"}]},"ts":"1733740929950"} 2024-12-09T10:42:09,951 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-09T10:42:09,951 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ec3eeaafd032b9a023fe519c0fcbb0c5, ASSIGN}] 2024-12-09T10:42:09,952 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ec3eeaafd032b9a023fe519c0fcbb0c5, ASSIGN 2024-12-09T10:42:09,953 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ec3eeaafd032b9a023fe519c0fcbb0c5, ASSIGN; state=OFFLINE, location=7ea43b39fc24,36101,1733740827678; forceNewPlan=false, retain=false 2024-12-09T10:42:10,103 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=ec3eeaafd032b9a023fe519c0fcbb0c5, regionState=OPENING, regionLocation=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:10,105 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ec3eeaafd032b9a023fe519c0fcbb0c5, ASSIGN because future has completed 2024-12-09T10:42:10,106 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE, hasLock=false; OpenRegionProcedure ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678}] 2024-12-09T10:42:10,260 INFO [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. 2024-12-09T10:42:10,260 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(7752): Opening region: {ENCODED => ec3eeaafd032b9a023fe519c0fcbb0c5, NAME => 'TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.', STARTKEY => '', ENDKEY => ''} 2024-12-09T10:42:10,261 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees ec3eeaafd032b9a023fe519c0fcbb0c5 2024-12-09T10:42:10,261 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T10:42:10,261 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(7794): checking encryption for ec3eeaafd032b9a023fe519c0fcbb0c5 2024-12-09T10:42:10,261 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(7797): checking classloading for ec3eeaafd032b9a023fe519c0fcbb0c5 2024-12-09T10:42:10,262 INFO [StoreOpener-ec3eeaafd032b9a023fe519c0fcbb0c5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region ec3eeaafd032b9a023fe519c0fcbb0c5 2024-12-09T10:42:10,263 INFO [StoreOpener-ec3eeaafd032b9a023fe519c0fcbb0c5-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T10:42:10,263 INFO [StoreOpener-ec3eeaafd032b9a023fe519c0fcbb0c5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ec3eeaafd032b9a023fe519c0fcbb0c5 columnFamilyName A 2024-12-09T10:42:10,263 DEBUG [StoreOpener-ec3eeaafd032b9a023fe519c0fcbb0c5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:10,264 INFO [StoreOpener-ec3eeaafd032b9a023fe519c0fcbb0c5-1 {}] regionserver.HStore(327): Store=ec3eeaafd032b9a023fe519c0fcbb0c5/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T10:42:10,264 INFO [StoreOpener-ec3eeaafd032b9a023fe519c0fcbb0c5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region ec3eeaafd032b9a023fe519c0fcbb0c5 2024-12-09T10:42:10,265 INFO [StoreOpener-ec3eeaafd032b9a023fe519c0fcbb0c5-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T10:42:10,265 INFO [StoreOpener-ec3eeaafd032b9a023fe519c0fcbb0c5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ec3eeaafd032b9a023fe519c0fcbb0c5 columnFamilyName B 2024-12-09T10:42:10,265 DEBUG [StoreOpener-ec3eeaafd032b9a023fe519c0fcbb0c5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:10,265 INFO [StoreOpener-ec3eeaafd032b9a023fe519c0fcbb0c5-1 {}] regionserver.HStore(327): Store=ec3eeaafd032b9a023fe519c0fcbb0c5/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T10:42:10,265 INFO [StoreOpener-ec3eeaafd032b9a023fe519c0fcbb0c5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region ec3eeaafd032b9a023fe519c0fcbb0c5 2024-12-09T10:42:10,266 INFO [StoreOpener-ec3eeaafd032b9a023fe519c0fcbb0c5-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T10:42:10,266 INFO [StoreOpener-ec3eeaafd032b9a023fe519c0fcbb0c5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ec3eeaafd032b9a023fe519c0fcbb0c5 columnFamilyName C 2024-12-09T10:42:10,266 DEBUG [StoreOpener-ec3eeaafd032b9a023fe519c0fcbb0c5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:10,267 INFO [StoreOpener-ec3eeaafd032b9a023fe519c0fcbb0c5-1 {}] regionserver.HStore(327): Store=ec3eeaafd032b9a023fe519c0fcbb0c5/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T10:42:10,267 INFO [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. 2024-12-09T10:42:10,267 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1038): replaying wal for ec3eeaafd032b9a023fe519c0fcbb0c5 2024-12-09T10:42:10,267 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5 2024-12-09T10:42:10,268 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5 2024-12-09T10:42:10,268 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1048): stopping wal replay for ec3eeaafd032b9a023fe519c0fcbb0c5 2024-12-09T10:42:10,268 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1060): Cleaning up temporary data for ec3eeaafd032b9a023fe519c0fcbb0c5 2024-12-09T10:42:10,269 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-09T10:42:10,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-12-09T10:42:10,270 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1093): writing seq id for ec3eeaafd032b9a023fe519c0fcbb0c5 2024-12-09T10:42:10,272 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T10:42:10,272 INFO [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1114): Opened ec3eeaafd032b9a023fe519c0fcbb0c5; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70271457, jitterRate=0.04712630808353424}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-09T10:42:10,272 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ec3eeaafd032b9a023fe519c0fcbb0c5 2024-12-09T10:42:10,273 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1006): Region open journal for ec3eeaafd032b9a023fe519c0fcbb0c5: Running coprocessor pre-open hook at 1733740930261Writing region info on filesystem at 1733740930261Initializing all the Stores at 1733740930262 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733740930262Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733740930262Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733740930262Cleaning up temporary data from old regions at 1733740930268 (+6 ms)Running coprocessor post-open hooks at 1733740930272 (+4 ms)Region opened successfully at 1733740930272 2024-12-09T10:42:10,273 INFO [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., pid=87, masterSystemTime=1733740930257 2024-12-09T10:42:10,275 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. 2024-12-09T10:42:10,275 INFO [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. 2024-12-09T10:42:10,275 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=ec3eeaafd032b9a023fe519c0fcbb0c5, regionState=OPEN, openSeqNum=2, regionLocation=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:10,277 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=87, ppid=86, state=RUNNABLE, hasLock=false; OpenRegionProcedure ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 because future has completed 2024-12-09T10:42:10,277 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35095 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=7ea43b39fc24,36101,1733740827678, table=TestAcidGuarantees, region=ec3eeaafd032b9a023fe519c0fcbb0c5. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-09T10:42:10,279 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-12-09T10:42:10,279 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; OpenRegionProcedure ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 in 172 msec 2024-12-09T10:42:10,280 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=86, resume processing ppid=85 2024-12-09T10:42:10,280 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, ppid=85, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ec3eeaafd032b9a023fe519c0fcbb0c5, ASSIGN in 328 msec 2024-12-09T10:42:10,281 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-09T10:42:10,281 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733740930281"}]},"ts":"1733740930281"} 2024-12-09T10:42:10,282 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-09T10:42:10,283 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-09T10:42:10,284 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1520 sec 2024-12-09T10:42:11,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-12-09T10:42:11,290 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-09T10:42:11,291 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@634c1d3d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:42:11,291 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ea43b39fc24,35095,-1 for getting cluster id 2024-12-09T10:42:11,291 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T10:42:11,292 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0945b5ae-b46b-4b8f-af48-56b0bb92df09' 2024-12-09T10:42:11,293 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T10:42:11,293 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0945b5ae-b46b-4b8f-af48-56b0bb92df09" 2024-12-09T10:42:11,293 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67dd6e91, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:42:11,293 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ea43b39fc24,35095,-1] 2024-12-09T10:42:11,293 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T10:42:11,293 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:42:11,294 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59602, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T10:42:11,295 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55a7775d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:42:11,295 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T10:42:11,296 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ea43b39fc24,36101,1733740827678, seqNum=-1] 2024-12-09T10:42:11,296 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T10:42:11,297 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50908, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T10:42:11,297 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-09T10:42:11,298 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 7ea43b39fc24,35095,1733740826918 2024-12-09T10:42:11,298 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@6e5fb9f3 2024-12-09T10:42:11,298 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-09T10:42:11,299 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59608, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-09T10:42:11,300 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-09T10:42:11,300 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T10:42:11,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] procedure2.ProcedureExecutor(1139): Stored pid=88, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-09T10:42:11,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742156_1332 (size=1097) 2024-12-09T10:42:11,712 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-09T10:42:11,712 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-09T10:42:11,714 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-09T10:42:11,716 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ec3eeaafd032b9a023fe519c0fcbb0c5, REOPEN/MOVE}] 2024-12-09T10:42:11,717 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ec3eeaafd032b9a023fe519c0fcbb0c5, REOPEN/MOVE 2024-12-09T10:42:11,717 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=ec3eeaafd032b9a023fe519c0fcbb0c5, regionState=CLOSING, regionLocation=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:11,719 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ec3eeaafd032b9a023fe519c0fcbb0c5, REOPEN/MOVE because future has completed 2024-12-09T10:42:11,719 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-09T10:42:11,719 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE, hasLock=false; CloseRegionProcedure ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678}] 2024-12-09T10:42:11,871 INFO [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(122): Close ec3eeaafd032b9a023fe519c0fcbb0c5 2024-12-09T10:42:11,871 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-09T10:42:11,871 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1722): Closing ec3eeaafd032b9a023fe519c0fcbb0c5, disabling compactions & flushes 2024-12-09T10:42:11,871 INFO [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. 2024-12-09T10:42:11,871 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. 2024-12-09T10:42:11,871 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. after waiting 0 ms 2024-12-09T10:42:11,871 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. 2024-12-09T10:42:11,875 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-09T10:42:11,875 INFO [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. 2024-12-09T10:42:11,875 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1676): Region close journal for ec3eeaafd032b9a023fe519c0fcbb0c5: Waiting for close lock at 1733740931871Running coprocessor pre-close hooks at 1733740931871Disabling compacts and flushes for region at 1733740931871Disabling writes for close at 1733740931871Writing region close event to WAL at 1733740931872 (+1 ms)Running coprocessor post-close hooks at 1733740931875 (+3 ms)Closed at 1733740931875 2024-12-09T10:42:11,875 WARN [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegionServer(3299): Not adding moved region record: ec3eeaafd032b9a023fe519c0fcbb0c5 to self. 2024-12-09T10:42:11,877 INFO [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(157): Closed ec3eeaafd032b9a023fe519c0fcbb0c5 2024-12-09T10:42:11,878 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=ec3eeaafd032b9a023fe519c0fcbb0c5, regionState=CLOSED 2024-12-09T10:42:11,879 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=91, ppid=90, state=RUNNABLE, hasLock=false; CloseRegionProcedure ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 because future has completed 2024-12-09T10:42:11,881 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-12-09T10:42:11,881 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; CloseRegionProcedure ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 in 160 msec 2024-12-09T10:42:11,881 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ec3eeaafd032b9a023fe519c0fcbb0c5, REOPEN/MOVE; state=CLOSED, location=7ea43b39fc24,36101,1733740827678; forceNewPlan=false, retain=true 2024-12-09T10:42:12,032 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=ec3eeaafd032b9a023fe519c0fcbb0c5, regionState=OPENING, regionLocation=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:12,033 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ec3eeaafd032b9a023fe519c0fcbb0c5, REOPEN/MOVE because future has completed 2024-12-09T10:42:12,034 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=90, state=RUNNABLE, hasLock=false; OpenRegionProcedure ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678}] 2024-12-09T10:42:12,188 INFO [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. 2024-12-09T10:42:12,188 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7752): Opening region: {ENCODED => ec3eeaafd032b9a023fe519c0fcbb0c5, NAME => 'TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.', STARTKEY => '', ENDKEY => ''} 2024-12-09T10:42:12,188 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees ec3eeaafd032b9a023fe519c0fcbb0c5 2024-12-09T10:42:12,188 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T10:42:12,188 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7794): checking encryption for ec3eeaafd032b9a023fe519c0fcbb0c5 2024-12-09T10:42:12,188 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7797): checking classloading for ec3eeaafd032b9a023fe519c0fcbb0c5 2024-12-09T10:42:12,190 INFO [StoreOpener-ec3eeaafd032b9a023fe519c0fcbb0c5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region ec3eeaafd032b9a023fe519c0fcbb0c5 2024-12-09T10:42:12,190 INFO [StoreOpener-ec3eeaafd032b9a023fe519c0fcbb0c5-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T10:42:12,191 INFO [StoreOpener-ec3eeaafd032b9a023fe519c0fcbb0c5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ec3eeaafd032b9a023fe519c0fcbb0c5 columnFamilyName A 2024-12-09T10:42:12,192 DEBUG [StoreOpener-ec3eeaafd032b9a023fe519c0fcbb0c5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:12,192 INFO [StoreOpener-ec3eeaafd032b9a023fe519c0fcbb0c5-1 {}] regionserver.HStore(327): Store=ec3eeaafd032b9a023fe519c0fcbb0c5/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T10:42:12,192 INFO [StoreOpener-ec3eeaafd032b9a023fe519c0fcbb0c5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region ec3eeaafd032b9a023fe519c0fcbb0c5 2024-12-09T10:42:12,193 INFO [StoreOpener-ec3eeaafd032b9a023fe519c0fcbb0c5-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T10:42:12,193 INFO [StoreOpener-ec3eeaafd032b9a023fe519c0fcbb0c5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ec3eeaafd032b9a023fe519c0fcbb0c5 columnFamilyName B 2024-12-09T10:42:12,193 DEBUG [StoreOpener-ec3eeaafd032b9a023fe519c0fcbb0c5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:12,193 INFO [StoreOpener-ec3eeaafd032b9a023fe519c0fcbb0c5-1 {}] regionserver.HStore(327): Store=ec3eeaafd032b9a023fe519c0fcbb0c5/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T10:42:12,193 INFO [StoreOpener-ec3eeaafd032b9a023fe519c0fcbb0c5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region ec3eeaafd032b9a023fe519c0fcbb0c5 2024-12-09T10:42:12,194 INFO [StoreOpener-ec3eeaafd032b9a023fe519c0fcbb0c5-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T10:42:12,194 INFO [StoreOpener-ec3eeaafd032b9a023fe519c0fcbb0c5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ec3eeaafd032b9a023fe519c0fcbb0c5 columnFamilyName C 2024-12-09T10:42:12,194 DEBUG [StoreOpener-ec3eeaafd032b9a023fe519c0fcbb0c5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:12,194 INFO [StoreOpener-ec3eeaafd032b9a023fe519c0fcbb0c5-1 {}] regionserver.HStore(327): Store=ec3eeaafd032b9a023fe519c0fcbb0c5/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T10:42:12,194 INFO [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. 2024-12-09T10:42:12,194 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1038): replaying wal for ec3eeaafd032b9a023fe519c0fcbb0c5 2024-12-09T10:42:12,195 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5 2024-12-09T10:42:12,196 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5 2024-12-09T10:42:12,196 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1048): stopping wal replay for ec3eeaafd032b9a023fe519c0fcbb0c5 2024-12-09T10:42:12,196 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1060): Cleaning up temporary data for ec3eeaafd032b9a023fe519c0fcbb0c5 2024-12-09T10:42:12,197 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-09T10:42:12,198 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1093): writing seq id for ec3eeaafd032b9a023fe519c0fcbb0c5 2024-12-09T10:42:12,198 INFO [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1114): Opened ec3eeaafd032b9a023fe519c0fcbb0c5; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73290269, jitterRate=0.09211011230945587}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-09T10:42:12,198 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ec3eeaafd032b9a023fe519c0fcbb0c5 2024-12-09T10:42:12,199 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1006): Region open journal for ec3eeaafd032b9a023fe519c0fcbb0c5: Running coprocessor pre-open hook at 1733740932189Writing region info on filesystem at 1733740932189Initializing all the Stores at 1733740932189Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733740932189Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733740932189Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733740932189Cleaning up temporary data from old regions at 1733740932196 (+7 ms)Running coprocessor post-open hooks at 1733740932198 (+2 ms)Region opened successfully at 1733740932199 (+1 ms) 2024-12-09T10:42:12,200 INFO [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., pid=92, masterSystemTime=1733740932185 2024-12-09T10:42:12,201 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. 2024-12-09T10:42:12,201 INFO [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. 2024-12-09T10:42:12,202 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=ec3eeaafd032b9a023fe519c0fcbb0c5, regionState=OPEN, openSeqNum=5, regionLocation=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:12,203 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=90, state=RUNNABLE, hasLock=false; OpenRegionProcedure ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 because future has completed 2024-12-09T10:42:12,205 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=90 2024-12-09T10:42:12,205 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=90, state=SUCCESS, hasLock=false; OpenRegionProcedure ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 in 171 msec 2024-12-09T10:42:12,207 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=89 2024-12-09T10:42:12,207 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=89, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ec3eeaafd032b9a023fe519c0fcbb0c5, REOPEN/MOVE in 489 msec 2024-12-09T10:42:12,208 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-12-09T10:42:12,209 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 493 msec 2024-12-09T10:42:12,210 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 909 msec 2024-12-09T10:42:12,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-12-09T10:42:12,211 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-09T10:42:12,212 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45a439d4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:42:12,212 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ea43b39fc24,35095,-1 for getting cluster id 2024-12-09T10:42:12,212 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T10:42:12,213 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0945b5ae-b46b-4b8f-af48-56b0bb92df09' 2024-12-09T10:42:12,213 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T10:42:12,213 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0945b5ae-b46b-4b8f-af48-56b0bb92df09" 2024-12-09T10:42:12,213 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1da1c474, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:42:12,213 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ea43b39fc24,35095,-1] 2024-12-09T10:42:12,213 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T10:42:12,214 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:42:12,214 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59626, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T10:42:12,215 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2eccc0c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:42:12,215 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31661a39, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:42:12,215 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ea43b39fc24,35095,-1 for getting cluster id 2024-12-09T10:42:12,215 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T10:42:12,216 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0945b5ae-b46b-4b8f-af48-56b0bb92df09' 2024-12-09T10:42:12,216 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T10:42:12,216 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0945b5ae-b46b-4b8f-af48-56b0bb92df09" 2024-12-09T10:42:12,216 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75c8a11c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:42:12,216 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ea43b39fc24,35095,-1] 2024-12-09T10:42:12,216 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T10:42:12,217 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:42:12,217 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59646, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T10:42:12,218 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7dffe94e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:42:12,218 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f2dd5c2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:42:12,218 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ea43b39fc24,35095,-1 for getting cluster id 2024-12-09T10:42:12,218 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T10:42:12,219 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0945b5ae-b46b-4b8f-af48-56b0bb92df09' 2024-12-09T10:42:12,219 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T10:42:12,219 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0945b5ae-b46b-4b8f-af48-56b0bb92df09" 2024-12-09T10:42:12,219 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2aae255e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:42:12,219 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ea43b39fc24,35095,-1] 2024-12-09T10:42:12,219 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T10:42:12,219 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:42:12,220 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59664, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T10:42:12,220 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@138557df, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:42:12,221 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ca4559e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:42:12,221 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ea43b39fc24,35095,-1 for getting cluster id 2024-12-09T10:42:12,221 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T10:42:12,222 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0945b5ae-b46b-4b8f-af48-56b0bb92df09' 2024-12-09T10:42:12,222 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T10:42:12,222 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0945b5ae-b46b-4b8f-af48-56b0bb92df09" 2024-12-09T10:42:12,222 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@259b3525, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:42:12,222 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ea43b39fc24,35095,-1] 2024-12-09T10:42:12,222 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T10:42:12,222 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:42:12,223 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59694, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T10:42:12,223 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21572804, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:42:12,224 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79231b2f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:42:12,224 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ea43b39fc24,35095,-1 for getting cluster id 2024-12-09T10:42:12,224 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T10:42:12,224 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0945b5ae-b46b-4b8f-af48-56b0bb92df09' 2024-12-09T10:42:12,224 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T10:42:12,225 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0945b5ae-b46b-4b8f-af48-56b0bb92df09" 2024-12-09T10:42:12,225 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@761eab8c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:42:12,225 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ea43b39fc24,35095,-1] 2024-12-09T10:42:12,225 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T10:42:12,225 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:42:12,226 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59714, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T10:42:12,226 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c73e89b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:42:12,227 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cbc52a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:42:12,227 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ea43b39fc24,35095,-1 for getting cluster id 2024-12-09T10:42:12,227 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T10:42:12,228 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0945b5ae-b46b-4b8f-af48-56b0bb92df09' 2024-12-09T10:42:12,228 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T10:42:12,228 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0945b5ae-b46b-4b8f-af48-56b0bb92df09" 2024-12-09T10:42:12,228 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41dc9788, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:42:12,228 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ea43b39fc24,35095,-1] 2024-12-09T10:42:12,228 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T10:42:12,228 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:42:12,229 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59732, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T10:42:12,229 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ddcdf6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:42:12,230 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62f5817b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:42:12,230 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ea43b39fc24,35095,-1 for getting cluster id 2024-12-09T10:42:12,230 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T10:42:12,231 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0945b5ae-b46b-4b8f-af48-56b0bb92df09' 2024-12-09T10:42:12,231 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T10:42:12,231 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0945b5ae-b46b-4b8f-af48-56b0bb92df09" 2024-12-09T10:42:12,231 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@722de190, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:42:12,231 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ea43b39fc24,35095,-1] 2024-12-09T10:42:12,231 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T10:42:12,231 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:42:12,232 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59742, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T10:42:12,232 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b82b5a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:42:12,233 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65cae09d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:42:12,233 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ea43b39fc24,35095,-1 for getting cluster id 2024-12-09T10:42:12,233 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T10:42:12,233 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0945b5ae-b46b-4b8f-af48-56b0bb92df09' 2024-12-09T10:42:12,234 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T10:42:12,234 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0945b5ae-b46b-4b8f-af48-56b0bb92df09" 2024-12-09T10:42:12,234 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@708a6cec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:42:12,234 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ea43b39fc24,35095,-1] 2024-12-09T10:42:12,234 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T10:42:12,234 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:42:12,235 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59760, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T10:42:12,235 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6dcfcc17, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:42:12,236 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b3e43b6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:42:12,236 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ea43b39fc24,35095,-1 for getting cluster id 2024-12-09T10:42:12,236 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T10:42:12,237 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0945b5ae-b46b-4b8f-af48-56b0bb92df09' 2024-12-09T10:42:12,237 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T10:42:12,237 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0945b5ae-b46b-4b8f-af48-56b0bb92df09" 2024-12-09T10:42:12,237 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e5bf869, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:42:12,237 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ea43b39fc24,35095,-1] 2024-12-09T10:42:12,237 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T10:42:12,237 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:42:12,238 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59788, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T10:42:12,238 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4db75b40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:42:12,239 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ab023db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:42:12,239 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ea43b39fc24,35095,-1 for getting cluster id 2024-12-09T10:42:12,239 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T10:42:12,240 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0945b5ae-b46b-4b8f-af48-56b0bb92df09' 2024-12-09T10:42:12,240 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T10:42:12,240 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0945b5ae-b46b-4b8f-af48-56b0bb92df09" 2024-12-09T10:42:12,240 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1637d3aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:42:12,240 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ea43b39fc24,35095,-1] 2024-12-09T10:42:12,240 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T10:42:12,240 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:42:12,241 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59804, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T10:42:12,241 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52f6826f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:42:12,242 DEBUG [Thread-1527 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T10:42:12,242 DEBUG [Thread-1540 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T10:42:12,242 DEBUG [Thread-1536 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T10:42:12,242 DEBUG [Thread-1538 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T10:42:12,242 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-09T10:42:12,242 DEBUG [Thread-1534 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T10:42:12,243 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ea43b39fc24,36101,1733740827678, seqNum=-1] 2024-12-09T10:42:12,243 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ea43b39fc24,36101,1733740827678, seqNum=-1] 2024-12-09T10:42:12,243 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T10:42:12,243 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T10:42:12,243 DEBUG [Thread-1521 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T10:42:12,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] procedure2.ProcedureExecutor(1139): Stored pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees 2024-12-09T10:42:12,243 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ea43b39fc24,36101,1733740827678, seqNum=-1] 2024-12-09T10:42:12,244 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T10:42:12,244 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ea43b39fc24,36101,1733740827678, seqNum=-1] 2024-12-09T10:42:12,244 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T10:42:12,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-09T10:42:12,244 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50916, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T10:42:12,244 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50928, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T10:42:12,244 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ea43b39fc24,36101,1733740827678, seqNum=-1] 2024-12-09T10:42:12,245 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ea43b39fc24,36101,1733740827678, seqNum=-1] 2024-12-09T10:42:12,245 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T10:42:12,245 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50940, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T10:42:12,245 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T10:42:12,245 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50934, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T10:42:12,245 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50942, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T10:42:12,246 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50952, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T10:42:12,246 DEBUG [Thread-1523 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T10:42:12,246 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T10:42:12,246 DEBUG [Thread-1525 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T10:42:12,246 DEBUG [Thread-1532 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T10:42:12,246 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ea43b39fc24,36101,1733740827678, seqNum=-1] 2024-12-09T10:42:12,246 DEBUG [Thread-1529 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T10:42:12,247 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T10:42:12,247 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T10:42:12,247 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5] 2024-12-09T10:42:12,247 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T10:42:12,247 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ea43b39fc24,36101,1733740827678, seqNum=-1] 2024-12-09T10:42:12,248 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T10:42:12,248 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5] 2024-12-09T10:42:12,248 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50954, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T10:42:12,249 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50962, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T10:42:12,250 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5] 2024-12-09T10:42:12,250 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5] 2024-12-09T10:42:12,251 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5] 2024-12-09T10:42:12,251 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5] 2024-12-09T10:42:12,252 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ea43b39fc24,36101,1733740827678, seqNum=-1] 2024-12-09T10:42:12,252 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ea43b39fc24,36101,1733740827678, seqNum=-1] 2024-12-09T10:42:12,252 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T10:42:12,252 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T10:42:12,253 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5] 2024-12-09T10:42:12,253 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5] 2024-12-09T10:42:12,254 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50974, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T10:42:12,254 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50968, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T10:42:12,256 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5] 2024-12-09T10:42:12,257 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5] 2024-12-09T10:42:12,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(8855): Flush requested on ec3eeaafd032b9a023fe519c0fcbb0c5 2024-12-09T10:42:12,258 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ec3eeaafd032b9a023fe519c0fcbb0c5 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-09T10:42:12,259 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec3eeaafd032b9a023fe519c0fcbb0c5, store=A 2024-12-09T10:42:12,259 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:42:12,259 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec3eeaafd032b9a023fe519c0fcbb0c5, store=B 2024-12-09T10:42:12,259 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:42:12,259 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec3eeaafd032b9a023fe519c0fcbb0c5, store=C 2024-12-09T10:42:12,259 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:42:12,283 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209200df86faf4246319f3e78467a004d05_ec3eeaafd032b9a023fe519c0fcbb0c5 is 50, key is test_row_0/A:col10/1733740932257/Put/seqid=0 2024-12-09T10:42:12,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742157_1333 (size=19474) 2024-12-09T10:42:12,303 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:12,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50954 deadline: 1733740992290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:12,305 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:12,305 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:12,305 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:12,305 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:12,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50962 deadline: 1733740992291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:12,305 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:12,305 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:12,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50928 deadline: 1733740992299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:12,305 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:12,305 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:12,305 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733740932271, max timestamp=1733740932302Num uniques 859; 2024-12-09T10:42:12,305 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-09T10:42:12,306 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733740932271, max timestamp=1733740932302Num uniques 859; 2024-12-09T10:42:12,306 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-09T10:42:12,306 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733740932271, max timestamp=1733740932302Num uniques 859; 2024-12-09T10:42:12,306 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-09T10:42:12,306 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:12,306 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:12,306 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:12,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-09T10:42:12,369 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:12,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50942 deadline: 1733740992360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:12,369 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:12,369 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:12,369 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:12,388 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:12,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50974 deadline: 1733740992369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:12,389 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:12,389 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:12,389 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:12,400 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-09T10:42:12,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. 2024-12-09T10:42:12,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. as already flushing 2024-12-09T10:42:12,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. 2024-12-09T10:42:12,401 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => ec3eeaafd032b9a023fe519c0fcbb0c5, NAME => 'TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:12,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => ec3eeaafd032b9a023fe519c0fcbb0c5, NAME => 'TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:12,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ec3eeaafd032b9a023fe519c0fcbb0c5, NAME => 'TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ec3eeaafd032b9a023fe519c0fcbb0c5, NAME => 'TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:12,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:12,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50954 deadline: 1733740992409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:12,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:12,419 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:12,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50962 deadline: 1733740992409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:12,419 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:12,419 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:12,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:12,419 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:12,420 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:12,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50928 deadline: 1733740992409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:12,420 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:12,420 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:12,420 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:12,420 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:12,487 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:12,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50942 deadline: 1733740992479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:12,488 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:12,488 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:12,488 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:12,508 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:12,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50974 deadline: 1733740992499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:12,509 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:12,509 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:12,509 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:12,553 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-09T10:42:12,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. 2024-12-09T10:42:12,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. as already flushing 2024-12-09T10:42:12,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. 2024-12-09T10:42:12,554 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => ec3eeaafd032b9a023fe519c0fcbb0c5, NAME => 'TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:12,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => ec3eeaafd032b9a023fe519c0fcbb0c5, NAME => 'TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:12,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ec3eeaafd032b9a023fe519c0fcbb0c5, NAME => 'TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ec3eeaafd032b9a023fe519c0fcbb0c5, NAME => 'TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:12,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-09T10:42:12,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:12,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50962 deadline: 1733740992630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:12,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:12,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50954 deadline: 1733740992630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:12,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:12,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50928 deadline: 1733740992630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:12,639 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:12,639 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:12,639 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:12,639 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:12,639 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:12,639 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:12,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:12,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:12,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:12,697 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:12,701 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209200df86faf4246319f3e78467a004d05_ec3eeaafd032b9a023fe519c0fcbb0c5 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209200df86faf4246319f3e78467a004d05_ec3eeaafd032b9a023fe519c0fcbb0c5 2024-12-09T10:42:12,702 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/A/109a22e8b6744c7d82d4e0268b1f411c, store: [table=TestAcidGuarantees family=A region=ec3eeaafd032b9a023fe519c0fcbb0c5] 2024-12-09T10:42:12,703 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/A/109a22e8b6744c7d82d4e0268b1f411c is 175, key is test_row_0/A:col10/1733740932257/Put/seqid=0 2024-12-09T10:42:12,706 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-09T10:42:12,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. 2024-12-09T10:42:12,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742158_1334 (size=56733) 2024-12-09T10:42:12,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. as already flushing 2024-12-09T10:42:12,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. 2024-12-09T10:42:12,707 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => ec3eeaafd032b9a023fe519c0fcbb0c5, NAME => 'TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:12,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => ec3eeaafd032b9a023fe519c0fcbb0c5, NAME => 'TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:12,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ec3eeaafd032b9a023fe519c0fcbb0c5, NAME => 'TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ec3eeaafd032b9a023fe519c0fcbb0c5, NAME => 'TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:12,710 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:12,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50942 deadline: 1733740992700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:12,711 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:12,711 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:12,711 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:12,728 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:12,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50974 deadline: 1733740992719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:12,729 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:12,729 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:12,729 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:12,860 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-09T10:42:12,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. 2024-12-09T10:42:12,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. as already flushing 2024-12-09T10:42:12,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. 2024-12-09T10:42:12,860 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => ec3eeaafd032b9a023fe519c0fcbb0c5, NAME => 'TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:12,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => ec3eeaafd032b9a023fe519c0fcbb0c5, NAME => 'TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:12,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ec3eeaafd032b9a023fe519c0fcbb0c5, NAME => 'TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ec3eeaafd032b9a023fe519c0fcbb0c5, NAME => 'TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:12,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-09T10:42:12,958 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:12,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50962 deadline: 1733740992949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:12,958 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:12,958 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:12,958 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:12,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50928 deadline: 1733740992949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:12,958 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:12,959 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:12,959 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:12,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50954 deadline: 1733740992949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:12,959 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:12,959 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:12,959 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:12,959 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:12,959 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:13,013 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-09T10:42:13,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. 2024-12-09T10:42:13,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. as already flushing 2024-12-09T10:42:13,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. 2024-12-09T10:42:13,013 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => ec3eeaafd032b9a023fe519c0fcbb0c5, NAME => 'TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:13,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => ec3eeaafd032b9a023fe519c0fcbb0c5, NAME => 'TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:13,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ec3eeaafd032b9a023fe519c0fcbb0c5, NAME => 'TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ec3eeaafd032b9a023fe519c0fcbb0c5, NAME => 'TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:13,032 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:13,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50942 deadline: 1733740993019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:13,033 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:13,033 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:13,033 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:13,051 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:13,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50974 deadline: 1733740993040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:13,052 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:13,052 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:13,052 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:13,107 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/A/109a22e8b6744c7d82d4e0268b1f411c 2024-12-09T10:42:13,133 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/B/5c0aa3f183f244bc9bf673c22c016d3a is 50, key is test_row_0/B:col10/1733740932257/Put/seqid=0 2024-12-09T10:42:13,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742159_1335 (size=12001) 2024-12-09T10:42:13,137 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/B/5c0aa3f183f244bc9bf673c22c016d3a 2024-12-09T10:42:13,162 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/C/20d65f890c4f46c19b993aabc4feda36 is 50, key is test_row_0/C:col10/1733740932257/Put/seqid=0 2024-12-09T10:42:13,166 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-09T10:42:13,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. 2024-12-09T10:42:13,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. as already flushing 2024-12-09T10:42:13,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. 2024-12-09T10:42:13,167 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => ec3eeaafd032b9a023fe519c0fcbb0c5, NAME => 'TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:13,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => ec3eeaafd032b9a023fe519c0fcbb0c5, NAME => 'TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:13,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ec3eeaafd032b9a023fe519c0fcbb0c5, NAME => 'TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ec3eeaafd032b9a023fe519c0fcbb0c5, NAME => 'TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:13,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742160_1336 (size=12001) 2024-12-09T10:42:13,180 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/C/20d65f890c4f46c19b993aabc4feda36 2024-12-09T10:42:13,190 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/A/109a22e8b6744c7d82d4e0268b1f411c as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/109a22e8b6744c7d82d4e0268b1f411c 2024-12-09T10:42:13,198 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/109a22e8b6744c7d82d4e0268b1f411c, entries=300, sequenceid=15, filesize=55.4 K 2024-12-09T10:42:13,199 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/B/5c0aa3f183f244bc9bf673c22c016d3a as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B/5c0aa3f183f244bc9bf673c22c016d3a 2024-12-09T10:42:13,203 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B/5c0aa3f183f244bc9bf673c22c016d3a, entries=150, sequenceid=15, filesize=11.7 K 2024-12-09T10:42:13,204 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/C/20d65f890c4f46c19b993aabc4feda36 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C/20d65f890c4f46c19b993aabc4feda36 2024-12-09T10:42:13,208 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C/20d65f890c4f46c19b993aabc4feda36, entries=150, sequenceid=15, filesize=11.7 K 2024-12-09T10:42:13,208 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for ec3eeaafd032b9a023fe519c0fcbb0c5 in 950ms, sequenceid=15, compaction requested=false 2024-12-09T10:42:13,209 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-09T10:42:13,209 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ec3eeaafd032b9a023fe519c0fcbb0c5: 2024-12-09T10:42:13,320 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-09T10:42:13,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. 2024-12-09T10:42:13,320 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2902): Flushing ec3eeaafd032b9a023fe519c0fcbb0c5 3/3 column families, dataSize=127.47 KB heapSize=355.69 KB 2024-12-09T10:42:13,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec3eeaafd032b9a023fe519c0fcbb0c5, store=A 2024-12-09T10:42:13,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T10:42:13,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec3eeaafd032b9a023fe519c0fcbb0c5, store=B 2024-12-09T10:42:13,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T10:42:13,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec3eeaafd032b9a023fe519c0fcbb0c5, store=C 2024-12-09T10:42:13,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T10:42:13,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209b3721b0eedb04ed99a72186a0a27ef40_ec3eeaafd032b9a023fe519c0fcbb0c5 is 50, key is test_row_0/A:col10/1733740932344/Put/seqid=0 2024-12-09T10:42:13,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742161_1337 (size=12154) 2024-12-09T10:42:13,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-09T10:42:13,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(8855): Flush requested on ec3eeaafd032b9a023fe519c0fcbb0c5 2024-12-09T10:42:13,475 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. as already flushing 2024-12-09T10:42:13,502 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:13,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50962 deadline: 1733740993492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:13,502 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:13,502 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:13,502 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:13,505 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:13,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50928 deadline: 1733740993502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:13,505 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:13,505 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:13,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50954 deadline: 1733740993502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:13,505 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:13,505 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:13,505 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:13,505 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:13,505 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:13,546 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:13,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50942 deadline: 1733740993540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:13,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:13,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:13,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:13,569 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:13,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50974 deadline: 1733740993559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:13,570 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:13,570 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:13,570 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:13,618 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:13,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50962 deadline: 1733740993610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:13,619 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:13,619 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:13,619 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:13,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50928 deadline: 1733740993610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:13,619 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:13,619 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:13,619 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:13,619 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:13,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50954 deadline: 1733740993610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:13,619 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:13,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:13,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:13,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:13,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:13,738 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209b3721b0eedb04ed99a72186a0a27ef40_ec3eeaafd032b9a023fe519c0fcbb0c5 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209b3721b0eedb04ed99a72186a0a27ef40_ec3eeaafd032b9a023fe519c0fcbb0c5 2024-12-09T10:42:13,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/A/6b2d64585f4c41dabb2b98b13a2c093a, store: [table=TestAcidGuarantees family=A region=ec3eeaafd032b9a023fe519c0fcbb0c5] 2024-12-09T10:42:13,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/A/6b2d64585f4c41dabb2b98b13a2c093a is 175, key is test_row_0/A:col10/1733740932344/Put/seqid=0 2024-12-09T10:42:13,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742162_1338 (size=30955) 2024-12-09T10:42:13,746 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=49, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/A/6b2d64585f4c41dabb2b98b13a2c093a 2024-12-09T10:42:13,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/B/a9462ad893a142239d9222053b5908cd is 50, key is test_row_0/B:col10/1733740932344/Put/seqid=0 2024-12-09T10:42:13,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742163_1339 (size=12001) 2024-12-09T10:42:13,766 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/B/a9462ad893a142239d9222053b5908cd 2024-12-09T10:42:13,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/C/162225dfd7c443eca45d98b39020b374 is 50, key is test_row_0/C:col10/1733740932344/Put/seqid=0 2024-12-09T10:42:13,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742164_1340 (size=12001) 2024-12-09T10:42:13,840 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:13,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50928 deadline: 1733740993830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:13,841 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:13,841 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:13,841 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:13,841 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:13,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50954 deadline: 1733740993830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:13,841 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:13,841 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:13,841 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:13,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50962 deadline: 1733740993834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:13,841 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:13,842 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:13,842 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:13,842 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:13,872 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-09T10:42:14,160 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:14,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50928 deadline: 1733740994149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:14,160 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:14,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50954 deadline: 1733740994150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:14,160 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:14,160 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:14,160 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:14,160 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:14,160 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:14,161 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:14,161 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:14,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50962 deadline: 1733740994150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:14,161 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:14,161 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:14,161 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:14,181 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/C/162225dfd7c443eca45d98b39020b374 2024-12-09T10:42:14,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/A/6b2d64585f4c41dabb2b98b13a2c093a as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/6b2d64585f4c41dabb2b98b13a2c093a 2024-12-09T10:42:14,189 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/6b2d64585f4c41dabb2b98b13a2c093a, entries=150, sequenceid=49, filesize=30.2 K 2024-12-09T10:42:14,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/B/a9462ad893a142239d9222053b5908cd as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B/a9462ad893a142239d9222053b5908cd 2024-12-09T10:42:14,194 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B/a9462ad893a142239d9222053b5908cd, entries=150, sequenceid=49, filesize=11.7 K 2024-12-09T10:42:14,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/C/162225dfd7c443eca45d98b39020b374 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C/162225dfd7c443eca45d98b39020b374 2024-12-09T10:42:14,198 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C/162225dfd7c443eca45d98b39020b374, entries=150, sequenceid=49, filesize=11.7 K 2024-12-09T10:42:14,198 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~355.64 KB/364176, currentSize=60.38 KB/61830 for ec3eeaafd032b9a023fe519c0fcbb0c5 in 878ms, sequenceid=49, compaction requested=false 2024-12-09T10:42:14,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2603): Flush status journal for ec3eeaafd032b9a023fe519c0fcbb0c5: 2024-12-09T10:42:14,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. 2024-12-09T10:42:14,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=94 2024-12-09T10:42:14,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.HMaster(4169): Remote procedure done, pid=94 2024-12-09T10:42:14,201 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-12-09T10:42:14,201 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9530 sec 2024-12-09T10:42:14,203 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees in 1.9600 sec 2024-12-09T10:42:14,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-09T10:42:14,390 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T10:42:14,391 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-09T10:42:14,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees 2024-12-09T10:42:14,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-09T10:42:14,394 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T10:42:14,395 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T10:42:14,395 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T10:42:14,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-09T10:42:14,548 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-09T10:42:14,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. 2024-12-09T10:42:14,548 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2902): Flushing ec3eeaafd032b9a023fe519c0fcbb0c5 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-09T10:42:14,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec3eeaafd032b9a023fe519c0fcbb0c5, store=A 2024-12-09T10:42:14,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:42:14,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec3eeaafd032b9a023fe519c0fcbb0c5, store=B 2024-12-09T10:42:14,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:42:14,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec3eeaafd032b9a023fe519c0fcbb0c5, store=C 2024-12-09T10:42:14,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:42:14,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209ba989f47e68e4864a5756e018f1529bc_ec3eeaafd032b9a023fe519c0fcbb0c5 is 50, key is test_row_0/A:col10/1733740933500/Put/seqid=0 2024-12-09T10:42:14,564 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. as already flushing 2024-12-09T10:42:14,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(8855): Flush requested on ec3eeaafd032b9a023fe519c0fcbb0c5 2024-12-09T10:42:14,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742165_1341 (size=12154) 2024-12-09T10:42:14,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:14,580 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209ba989f47e68e4864a5756e018f1529bc_ec3eeaafd032b9a023fe519c0fcbb0c5 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209ba989f47e68e4864a5756e018f1529bc_ec3eeaafd032b9a023fe519c0fcbb0c5 2024-12-09T10:42:14,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/A/865db025f7b64c9f8cc118aea566ff6e, store: [table=TestAcidGuarantees family=A region=ec3eeaafd032b9a023fe519c0fcbb0c5] 2024-12-09T10:42:14,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/A/865db025f7b64c9f8cc118aea566ff6e is 175, key is test_row_0/A:col10/1733740933500/Put/seqid=0 2024-12-09T10:42:14,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742166_1342 (size=30955) 2024-12-09T10:42:14,593 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=61, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/A/865db025f7b64c9f8cc118aea566ff6e 2024-12-09T10:42:14,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/B/6b0fad94b2e94bb38296072812e15691 is 50, key is test_row_0/B:col10/1733740933500/Put/seqid=0 2024-12-09T10:42:14,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742167_1343 (size=12001) 2024-12-09T10:42:14,664 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:14,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50942 deadline: 1733740994647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:14,664 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:14,664 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:14,664 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:14,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:14,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50974 deadline: 1733740994664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:14,680 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:14,680 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:14,680 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:14,680 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:14,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50928 deadline: 1733740994669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:14,680 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:14,680 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:14,680 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:14,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50954 deadline: 1733740994669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:14,680 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:14,681 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:14,681 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:14,681 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:14,681 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:14,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50962 deadline: 1733740994669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:14,681 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:14,681 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:14,681 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:14,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-09T10:42:14,779 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:14,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50942 deadline: 1733740994769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:14,779 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:14,779 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:14,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:14,796 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:14,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50974 deadline: 1733740994790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:14,797 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:14,797 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:14,797 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:15,001 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:15,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50942 deadline: 1733740994990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:15,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:15,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:15,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:15,007 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:15,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50974 deadline: 1733740995000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:15,007 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/B/6b0fad94b2e94bb38296072812e15691 2024-12-09T10:42:15,008 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:15,008 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:15,008 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:15,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/C/a844161280444e78b808ed6353abf0e6 is 50, key is test_row_0/C:col10/1733740933500/Put/seqid=0 2024-12-09T10:42:15,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-09T10:42:15,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742168_1344 (size=12001) 2024-12-09T10:42:15,321 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:15,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50942 deadline: 1733740995309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:15,322 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:15,322 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:15,322 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:15,331 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:15,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50974 deadline: 1733740995320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:15,331 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:15,331 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:15,331 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:15,428 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/C/a844161280444e78b808ed6353abf0e6 2024-12-09T10:42:15,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/A/865db025f7b64c9f8cc118aea566ff6e as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/865db025f7b64c9f8cc118aea566ff6e 2024-12-09T10:42:15,438 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/865db025f7b64c9f8cc118aea566ff6e, entries=150, sequenceid=61, filesize=30.2 K 2024-12-09T10:42:15,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/B/6b0fad94b2e94bb38296072812e15691 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B/6b0fad94b2e94bb38296072812e15691 2024-12-09T10:42:15,445 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B/6b0fad94b2e94bb38296072812e15691, entries=150, sequenceid=61, filesize=11.7 K 2024-12-09T10:42:15,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/C/a844161280444e78b808ed6353abf0e6 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C/a844161280444e78b808ed6353abf0e6 2024-12-09T10:42:15,450 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C/a844161280444e78b808ed6353abf0e6, entries=150, sequenceid=61, filesize=11.7 K 2024-12-09T10:42:15,451 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for ec3eeaafd032b9a023fe519c0fcbb0c5 in 902ms, sequenceid=61, compaction requested=true 2024-12-09T10:42:15,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2603): Flush status journal for ec3eeaafd032b9a023fe519c0fcbb0c5: 2024-12-09T10:42:15,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. 2024-12-09T10:42:15,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-12-09T10:42:15,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.HMaster(4169): Remote procedure done, pid=96 2024-12-09T10:42:15,454 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-12-09T10:42:15,454 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0570 sec 2024-12-09T10:42:15,456 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees in 1.0640 sec 2024-12-09T10:42:15,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-09T10:42:15,530 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T10:42:15,530 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-09T10:42:15,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-12-09T10:42:15,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-09T10:42:15,532 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T10:42:15,533 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T10:42:15,533 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T10:42:15,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-09T10:42:15,685 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-09T10:42:15,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. 2024-12-09T10:42:15,686 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing ec3eeaafd032b9a023fe519c0fcbb0c5 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-09T10:42:15,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec3eeaafd032b9a023fe519c0fcbb0c5, store=A 2024-12-09T10:42:15,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:42:15,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec3eeaafd032b9a023fe519c0fcbb0c5, store=B 2024-12-09T10:42:15,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:42:15,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec3eeaafd032b9a023fe519c0fcbb0c5, store=C 2024-12-09T10:42:15,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:42:15,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(8855): Flush requested on ec3eeaafd032b9a023fe519c0fcbb0c5 2024-12-09T10:42:15,696 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. as already flushing 2024-12-09T10:42:15,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120941cdda7319904ec6aacb431c5608c7af_ec3eeaafd032b9a023fe519c0fcbb0c5 is 50, key is test_row_0/A:col10/1733740934662/Put/seqid=0 2024-12-09T10:42:15,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742169_1345 (size=12154) 2024-12-09T10:42:15,728 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:15,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50954 deadline: 1733740995716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:15,728 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:15,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50962 deadline: 1733740995725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:15,729 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:15,729 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:15,729 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:15,729 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:15,729 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:15,729 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:15,740 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:15,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50928 deadline: 1733740995726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:15,740 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:15,740 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:15,740 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:15,841 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:15,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50942 deadline: 1733740995829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:15,842 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:15,842 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:15,842 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:15,847 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:15,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50974 deadline: 1733740995840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:15,848 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:15,848 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:15,848 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:15,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50954 deadline: 1733740995840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:15,848 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:15,848 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:15,848 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:15,848 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:15,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50962 deadline: 1733740995840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:15,848 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:15,848 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:15,848 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:15,848 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:15,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-09T10:42:15,862 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:15,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50928 deadline: 1733740995849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:15,863 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:15,863 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:15,863 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:16,060 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:16,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50954 deadline: 1733740996050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:16,060 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:16,060 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:16,060 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:16,062 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:16,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50962 deadline: 1733740996059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:16,063 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:16,063 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:16,063 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:16,075 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:16,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50928 deadline: 1733740996069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:16,076 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:16,076 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:16,076 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:16,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:16,116 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120941cdda7319904ec6aacb431c5608c7af_ec3eeaafd032b9a023fe519c0fcbb0c5 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120941cdda7319904ec6aacb431c5608c7af_ec3eeaafd032b9a023fe519c0fcbb0c5 2024-12-09T10:42:16,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/A/ee5ffe6bb7b1459ca5355cd0dcdeb66b, store: [table=TestAcidGuarantees family=A region=ec3eeaafd032b9a023fe519c0fcbb0c5] 2024-12-09T10:42:16,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/A/ee5ffe6bb7b1459ca5355cd0dcdeb66b is 175, key is test_row_0/A:col10/1733740934662/Put/seqid=0 2024-12-09T10:42:16,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742170_1346 (size=30955) 2024-12-09T10:42:16,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-09T10:42:16,374 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:16,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50962 deadline: 1733740996369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:16,374 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:16,374 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:16,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50954 deadline: 1733740996369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:16,374 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:16,374 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:16,374 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:16,374 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:16,374 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:16,387 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:16,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50928 deadline: 1733740996379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:16,387 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:16,387 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:16,387 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:16,527 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=81, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/A/ee5ffe6bb7b1459ca5355cd0dcdeb66b 2024-12-09T10:42:16,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/B/e05a4d8e7b73445f97adea5aa4b23f9e is 50, key is test_row_0/B:col10/1733740934662/Put/seqid=0 2024-12-09T10:42:16,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742171_1347 (size=12001) 2024-12-09T10:42:16,539 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/B/e05a4d8e7b73445f97adea5aa4b23f9e 2024-12-09T10:42:16,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/C/8ec70b07680a4f5bad18ed997afd9d10 is 50, key is test_row_0/C:col10/1733740934662/Put/seqid=0 2024-12-09T10:42:16,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742172_1348 (size=12001) 2024-12-09T10:42:16,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-09T10:42:16,869 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:16,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50942 deadline: 1733740996859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:16,870 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:16,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50974 deadline: 1733740996859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:16,870 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 7ea43b39fc24,36101,1733740827678 for 'test_row_0' in ec3eeaafd032b9a023fe519c0fcbb0c5 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2222 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T10:42:16,870 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 7ea43b39fc24,36101,1733740827678 for 'test_row_0' in ec3eeaafd032b9a023fe519c0fcbb0c5 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2206 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T10:42:16,871 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:16,871 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:16,871 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:16,871 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:16,871 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:16,871 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:16,897 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:16,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50962 deadline: 1733740996890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:16,897 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:16,898 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:16,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50954 deadline: 1733740996890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:16,898 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:16,898 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:16,898 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:16,898 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:16,898 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:16,907 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:16,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50928 deadline: 1733740996899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:16,907 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:16,907 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:16,907 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:16,951 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/C/8ec70b07680a4f5bad18ed997afd9d10 2024-12-09T10:42:16,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/A/ee5ffe6bb7b1459ca5355cd0dcdeb66b as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/ee5ffe6bb7b1459ca5355cd0dcdeb66b 2024-12-09T10:42:16,961 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/ee5ffe6bb7b1459ca5355cd0dcdeb66b, entries=150, sequenceid=81, filesize=30.2 K 2024-12-09T10:42:16,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/B/e05a4d8e7b73445f97adea5aa4b23f9e as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B/e05a4d8e7b73445f97adea5aa4b23f9e 2024-12-09T10:42:16,965 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B/e05a4d8e7b73445f97adea5aa4b23f9e, entries=150, sequenceid=81, filesize=11.7 K 2024-12-09T10:42:16,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/C/8ec70b07680a4f5bad18ed997afd9d10 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C/8ec70b07680a4f5bad18ed997afd9d10 2024-12-09T10:42:16,971 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C/8ec70b07680a4f5bad18ed997afd9d10, entries=150, sequenceid=81, filesize=11.7 K 2024-12-09T10:42:16,972 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for ec3eeaafd032b9a023fe519c0fcbb0c5 in 1286ms, sequenceid=81, compaction requested=true 2024-12-09T10:42:16,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for ec3eeaafd032b9a023fe519c0fcbb0c5: 2024-12-09T10:42:16,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. 2024-12-09T10:42:16,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-12-09T10:42:16,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-12-09T10:42:16,977 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-12-09T10:42:16,977 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4410 sec 2024-12-09T10:42:16,979 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 1.4470 sec 2024-12-09T10:42:17,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-09T10:42:17,680 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T10:42:17,680 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-09T10:42:17,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-12-09T10:42:17,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-09T10:42:17,682 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T10:42:17,683 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T10:42:17,683 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T10:42:17,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-09T10:42:17,836 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-09T10:42:17,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. 2024-12-09T10:42:17,837 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing ec3eeaafd032b9a023fe519c0fcbb0c5 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-09T10:42:17,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec3eeaafd032b9a023fe519c0fcbb0c5, store=A 2024-12-09T10:42:17,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:42:17,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec3eeaafd032b9a023fe519c0fcbb0c5, store=B 2024-12-09T10:42:17,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:42:17,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec3eeaafd032b9a023fe519c0fcbb0c5, store=C 2024-12-09T10:42:17,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:42:17,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412092075e703a46b405caf0aa82842eea445_ec3eeaafd032b9a023fe519c0fcbb0c5 is 50, key is test_row_0/A:col10/1733740935714/Put/seqid=0 2024-12-09T10:42:17,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742173_1349 (size=12154) 2024-12-09T10:42:17,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(8855): Flush requested on ec3eeaafd032b9a023fe519c0fcbb0c5 2024-12-09T10:42:17,915 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. as already flushing 2024-12-09T10:42:17,984 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:17,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50962 deadline: 1733740997970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:17,985 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:17,985 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:17,985 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:17,987 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733740937953, max timestamp=1733740937983Num uniques 859; 2024-12-09T10:42:17,987 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733740937953, max timestamp=1733740937983Num uniques 859; 2024-12-09T10:42:17,987 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T10:42:17,987 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T10:42:17,987 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733740937953, max timestamp=1733740937983Num uniques 859; 2024-12-09T10:42:17,987 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T10:42:18,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-09T10:42:18,073 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:18,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50928 deadline: 1733740998053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:18,074 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:18,074 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:18,074 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:18,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50954 deadline: 1733740998064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:18,074 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:18,074 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:18,074 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:18,074 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:18,107 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:18,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50962 deadline: 1733740998090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:18,108 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:18,108 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:18,108 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:18,190 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:18,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50954 deadline: 1733740998180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:18,191 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:18,191 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:18,191 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:18,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50928 deadline: 1733740998180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:18,191 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:18,191 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:18,191 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:18,192 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:18,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:18,252 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412092075e703a46b405caf0aa82842eea445_ec3eeaafd032b9a023fe519c0fcbb0c5 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412092075e703a46b405caf0aa82842eea445_ec3eeaafd032b9a023fe519c0fcbb0c5 2024-12-09T10:42:18,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/A/79aa314b74184b4e8cd8009eb011b43b, store: [table=TestAcidGuarantees family=A region=ec3eeaafd032b9a023fe519c0fcbb0c5] 2024-12-09T10:42:18,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/A/79aa314b74184b4e8cd8009eb011b43b is 175, key is test_row_0/A:col10/1733740935714/Put/seqid=0 2024-12-09T10:42:18,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742174_1350 (size=30955) 2024-12-09T10:42:18,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-09T10:42:18,332 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:18,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50962 deadline: 1733740998319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:18,333 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:18,333 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:18,333 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:18,418 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:18,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50954 deadline: 1733740998400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:18,419 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:18,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:18,419 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:18,419 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:18,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50928 deadline: 1733740998400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:18,419 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:18,419 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:18,419 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:18,652 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:18,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50962 deadline: 1733740998640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:18,653 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:18,653 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:18,653 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:18,658 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=93, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/A/79aa314b74184b4e8cd8009eb011b43b 2024-12-09T10:42:18,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/B/fb883fa67fae4f72a07d754226b50f0f is 50, key is test_row_0/B:col10/1733740935714/Put/seqid=0 2024-12-09T10:42:18,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742175_1351 (size=12001) 2024-12-09T10:42:18,739 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:18,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50954 deadline: 1733740998730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:18,739 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:18,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50928 deadline: 1733740998730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:18,739 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:18,739 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:18,739 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:18,740 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:18,740 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:18,740 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:18,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-09T10:42:18,892 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:18,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50974 deadline: 1733740998880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:18,893 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 7ea43b39fc24,36101,1733740827678 for 'test_row_0' in ec3eeaafd032b9a023fe519c0fcbb0c5 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4229 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T10:42:18,893 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:18,893 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:18,894 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:18,903 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:18,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50942 deadline: 1733740998890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:18,903 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 7ea43b39fc24,36101,1733740827678 for 'test_row_0' in ec3eeaafd032b9a023fe519c0fcbb0c5 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4256 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T10:42:18,904 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:18,904 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:18,904 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:19,071 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/B/fb883fa67fae4f72a07d754226b50f0f 2024-12-09T10:42:19,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/C/431e8973d54d4052911a079d70c72e48 is 50, key is test_row_0/C:col10/1733740935714/Put/seqid=0 2024-12-09T10:42:19,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742176_1352 (size=12001) 2024-12-09T10:42:19,172 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:19,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50962 deadline: 1733740999159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:19,173 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:19,173 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:19,173 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:19,257 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:19,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50954 deadline: 1733740999249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:19,257 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:19,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50928 deadline: 1733740999249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:19,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:19,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:19,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:19,257 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:19,257 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:19,257 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:19,486 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/C/431e8973d54d4052911a079d70c72e48 2024-12-09T10:42:19,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/A/79aa314b74184b4e8cd8009eb011b43b as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/79aa314b74184b4e8cd8009eb011b43b 2024-12-09T10:42:19,498 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/79aa314b74184b4e8cd8009eb011b43b, entries=150, sequenceid=93, filesize=30.2 K 2024-12-09T10:42:19,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/B/fb883fa67fae4f72a07d754226b50f0f as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B/fb883fa67fae4f72a07d754226b50f0f 2024-12-09T10:42:19,504 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B/fb883fa67fae4f72a07d754226b50f0f, entries=150, sequenceid=93, filesize=11.7 K 2024-12-09T10:42:19,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/C/431e8973d54d4052911a079d70c72e48 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C/431e8973d54d4052911a079d70c72e48 2024-12-09T10:42:19,517 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C/431e8973d54d4052911a079d70c72e48, entries=150, sequenceid=93, filesize=11.7 K 2024-12-09T10:42:19,518 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=120.76 KB/123660 for ec3eeaafd032b9a023fe519c0fcbb0c5 in 1681ms, sequenceid=93, compaction requested=true 2024-12-09T10:42:19,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for ec3eeaafd032b9a023fe519c0fcbb0c5: 2024-12-09T10:42:19,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. 2024-12-09T10:42:19,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-12-09T10:42:19,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-12-09T10:42:19,523 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-12-09T10:42:19,523 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8380 sec 2024-12-09T10:42:19,525 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 1.8430 sec 2024-12-09T10:42:19,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-09T10:42:19,830 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T10:42:19,830 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-09T10:42:19,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-12-09T10:42:19,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-09T10:42:19,832 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T10:42:19,833 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T10:42:19,833 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T10:42:19,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-09T10:42:19,985 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-09T10:42:19,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. 2024-12-09T10:42:19,986 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing ec3eeaafd032b9a023fe519c0fcbb0c5 3/3 column families, dataSize=120.76 KB heapSize=340.45 KB 2024-12-09T10:42:19,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec3eeaafd032b9a023fe519c0fcbb0c5, store=A 2024-12-09T10:42:19,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T10:42:19,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec3eeaafd032b9a023fe519c0fcbb0c5, store=B 2024-12-09T10:42:19,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T10:42:19,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec3eeaafd032b9a023fe519c0fcbb0c5, store=C 2024-12-09T10:42:19,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T10:42:19,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412091db9286d29694c74ba342e6c9be3bcd5_ec3eeaafd032b9a023fe519c0fcbb0c5 is 50, key is test_row_0/A:col10/1733740938063/Put/seqid=0 2024-12-09T10:42:19,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742177_1353 (size=12154) 2024-12-09T10:42:19,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:20,003 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412091db9286d29694c74ba342e6c9be3bcd5_ec3eeaafd032b9a023fe519c0fcbb0c5 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412091db9286d29694c74ba342e6c9be3bcd5_ec3eeaafd032b9a023fe519c0fcbb0c5 2024-12-09T10:42:20,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/A/b385f56d32984e1e930315290ecdc5d1, store: [table=TestAcidGuarantees family=A region=ec3eeaafd032b9a023fe519c0fcbb0c5] 2024-12-09T10:42:20,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/A/b385f56d32984e1e930315290ecdc5d1 is 175, key is test_row_0/A:col10/1733740938063/Put/seqid=0 2024-12-09T10:42:20,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742178_1354 (size=30955) 2024-12-09T10:42:20,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-09T10:42:20,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(8855): Flush requested on ec3eeaafd032b9a023fe519c0fcbb0c5 2024-12-09T10:42:20,199 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. as already flushing 2024-12-09T10:42:20,294 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:20,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50962 deadline: 1733741000285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:20,295 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:20,295 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:20,295 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:20,303 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:20,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50954 deadline: 1733741000286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:20,303 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:20,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50928 deadline: 1733741000286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:20,303 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:20,303 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:20,303 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:20,303 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:20,303 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:20,303 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:20,406 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:20,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50962 deadline: 1733741000400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:20,407 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:20,407 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:20,407 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:20,408 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=127, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/A/b385f56d32984e1e930315290ecdc5d1 2024-12-09T10:42:20,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/B/0bb5884d15904307ba4074f165fa274e is 50, key is test_row_0/B:col10/1733740938063/Put/seqid=0 2024-12-09T10:42:20,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742179_1355 (size=12001) 2024-12-09T10:42:20,424 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:20,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50954 deadline: 1733741000409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:20,425 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:20,425 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:20,425 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:20,425 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:20,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50928 deadline: 1733741000409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:20,425 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:20,425 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:20,425 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:20,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-09T10:42:20,632 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:20,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50962 deadline: 1733741000619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:20,632 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:20,632 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:20,632 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:20,639 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:20,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50954 deadline: 1733741000629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:20,639 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:20,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50928 deadline: 1733741000629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:20,639 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:20,639 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:20,639 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:20,639 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:20,639 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:20,639 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:20,820 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/B/0bb5884d15904307ba4074f165fa274e 2024-12-09T10:42:20,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/C/ac17b63080d040a29c2825c823ce4af6 is 50, key is test_row_0/C:col10/1733740938063/Put/seqid=0 2024-12-09T10:42:20,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742180_1356 (size=12001) 2024-12-09T10:42:20,953 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:20,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50962 deadline: 1733741000939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:20,953 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:20,953 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:20,953 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:20,959 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:20,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50954 deadline: 1733741000949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:20,960 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:20,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50928 deadline: 1733741000949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:20,960 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:20,960 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:20,960 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:20,960 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:20,960 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:20,960 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:20,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-09T10:42:21,232 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/C/ac17b63080d040a29c2825c823ce4af6 2024-12-09T10:42:21,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/A/b385f56d32984e1e930315290ecdc5d1 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/b385f56d32984e1e930315290ecdc5d1 2024-12-09T10:42:21,240 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/b385f56d32984e1e930315290ecdc5d1, entries=150, sequenceid=127, filesize=30.2 K 2024-12-09T10:42:21,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/B/0bb5884d15904307ba4074f165fa274e as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B/0bb5884d15904307ba4074f165fa274e 2024-12-09T10:42:21,244 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B/0bb5884d15904307ba4074f165fa274e, entries=150, sequenceid=127, filesize=11.7 K 2024-12-09T10:42:21,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/C/ac17b63080d040a29c2825c823ce4af6 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C/ac17b63080d040a29c2825c823ce4af6 2024-12-09T10:42:21,250 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C/ac17b63080d040a29c2825c823ce4af6, entries=150, sequenceid=127, filesize=11.7 K 2024-12-09T10:42:21,250 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~340.41 KB/348576, currentSize=60.38 KB/61830 for ec3eeaafd032b9a023fe519c0fcbb0c5 in 1264ms, sequenceid=127, compaction requested=true 2024-12-09T10:42:21,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for ec3eeaafd032b9a023fe519c0fcbb0c5: 2024-12-09T10:42:21,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. 2024-12-09T10:42:21,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-12-09T10:42:21,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-12-09T10:42:21,258 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-12-09T10:42:21,258 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4230 sec 2024-12-09T10:42:21,259 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 1.4270 sec 2024-12-09T10:42:21,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(8855): Flush requested on ec3eeaafd032b9a023fe519c0fcbb0c5 2024-12-09T10:42:21,462 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ec3eeaafd032b9a023fe519c0fcbb0c5 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-09T10:42:21,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec3eeaafd032b9a023fe519c0fcbb0c5, store=A 2024-12-09T10:42:21,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:42:21,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec3eeaafd032b9a023fe519c0fcbb0c5, store=B 2024-12-09T10:42:21,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:42:21,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec3eeaafd032b9a023fe519c0fcbb0c5, store=C 2024-12-09T10:42:21,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:42:21,469 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412095d49d24868ac4821bf5688478bb3b62d_ec3eeaafd032b9a023fe519c0fcbb0c5 is 50, key is test_row_0/A:col10/1733740940285/Put/seqid=0 2024-12-09T10:42:21,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742181_1357 (size=14794) 2024-12-09T10:42:21,550 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:21,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50928 deadline: 1733741001531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:21,550 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:21,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50954 deadline: 1733741001532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:21,551 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:21,551 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:21,551 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:21,551 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:21,551 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:21,551 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:21,551 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:21,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50962 deadline: 1733741001537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:21,551 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:21,551 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:21,551 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:21,670 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:21,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50928 deadline: 1733741001660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:21,671 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:21,671 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:21,671 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:21,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50954 deadline: 1733741001660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:21,671 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:21,671 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:21,671 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:21,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50962 deadline: 1733741001660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:21,671 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:21,671 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:21,672 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:21,672 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:21,672 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:21,873 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:21,878 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412095d49d24868ac4821bf5688478bb3b62d_ec3eeaafd032b9a023fe519c0fcbb0c5 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412095d49d24868ac4821bf5688478bb3b62d_ec3eeaafd032b9a023fe519c0fcbb0c5 2024-12-09T10:42:21,879 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/A/7fe5f09840f84199a2b78fe54d3c2347, store: [table=TestAcidGuarantees family=A region=ec3eeaafd032b9a023fe519c0fcbb0c5] 2024-12-09T10:42:21,880 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/A/7fe5f09840f84199a2b78fe54d3c2347 is 175, key is test_row_0/A:col10/1733740940285/Put/seqid=0 2024-12-09T10:42:21,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742182_1358 (size=39749) 2024-12-09T10:42:21,891 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=140, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/A/7fe5f09840f84199a2b78fe54d3c2347 2024-12-09T10:42:21,895 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:21,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50928 deadline: 1733741001881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:21,896 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:21,896 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:21,896 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:21,896 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:21,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50954 deadline: 1733741001881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:21,896 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:21,896 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:21,896 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:21,896 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:21,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50962 deadline: 1733741001881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:21,897 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:21,897 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:21,897 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:21,901 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/B/e76673facc9e4d2a84856331c080b386 is 50, key is test_row_0/B:col10/1733740940285/Put/seqid=0 2024-12-09T10:42:21,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742183_1359 (size=12151) 2024-12-09T10:42:21,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-09T10:42:21,980 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T10:42:21,981 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-09T10:42:21,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-12-09T10:42:21,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-09T10:42:21,984 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T10:42:21,985 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T10:42:21,985 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T10:42:22,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-09T10:42:22,138 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-09T10:42:22,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. 2024-12-09T10:42:22,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. as already flushing 2024-12-09T10:42:22,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. 2024-12-09T10:42:22,139 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => ec3eeaafd032b9a023fe519c0fcbb0c5, NAME => 'TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:22,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => ec3eeaafd032b9a023fe519c0fcbb0c5, NAME => 'TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:22,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ec3eeaafd032b9a023fe519c0fcbb0c5, NAME => 'TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ec3eeaafd032b9a023fe519c0fcbb0c5, NAME => 'TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:22,210 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:22,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50928 deadline: 1733741002199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:22,211 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:22,211 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:22,211 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:22,223 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:22,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50954 deadline: 1733741002210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:22,224 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:22,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50962 deadline: 1733741002210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:22,224 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:22,224 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:22,224 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:22,224 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:22,224 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:22,224 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:22,293 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-09T10:42:22,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. 2024-12-09T10:42:22,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. as already flushing 2024-12-09T10:42:22,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. 2024-12-09T10:42:22,293 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => ec3eeaafd032b9a023fe519c0fcbb0c5, NAME => 'TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:22,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => ec3eeaafd032b9a023fe519c0fcbb0c5, NAME => 'TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:22,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ec3eeaafd032b9a023fe519c0fcbb0c5, NAME => 'TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ec3eeaafd032b9a023fe519c0fcbb0c5, NAME => 'TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:22,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-09T10:42:22,313 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/B/e76673facc9e4d2a84856331c080b386 2024-12-09T10:42:22,337 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/C/f3bee8c13f544f2693a7e46e8bc888fd is 50, key is test_row_0/C:col10/1733740940285/Put/seqid=0 2024-12-09T10:42:22,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742184_1360 (size=12151) 2024-12-09T10:42:22,383 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/C/f3bee8c13f544f2693a7e46e8bc888fd 2024-12-09T10:42:22,388 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/A/7fe5f09840f84199a2b78fe54d3c2347 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/7fe5f09840f84199a2b78fe54d3c2347 2024-12-09T10:42:22,392 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/7fe5f09840f84199a2b78fe54d3c2347, entries=200, sequenceid=140, filesize=38.8 K 2024-12-09T10:42:22,393 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/B/e76673facc9e4d2a84856331c080b386 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B/e76673facc9e4d2a84856331c080b386 2024-12-09T10:42:22,407 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B/e76673facc9e4d2a84856331c080b386, entries=150, sequenceid=140, filesize=11.9 K 2024-12-09T10:42:22,408 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/C/f3bee8c13f544f2693a7e46e8bc888fd as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C/f3bee8c13f544f2693a7e46e8bc888fd 2024-12-09T10:42:22,418 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C/f3bee8c13f544f2693a7e46e8bc888fd, entries=150, sequenceid=140, filesize=11.9 K 2024-12-09T10:42:22,419 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for ec3eeaafd032b9a023fe519c0fcbb0c5 in 957ms, sequenceid=140, compaction requested=true 2024-12-09T10:42:22,419 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ec3eeaafd032b9a023fe519c0fcbb0c5: 2024-12-09T10:42:22,419 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 7 store files, 0 compacting, 7 eligible, 16 blocking 2024-12-09T10:42:22,421 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ec3eeaafd032b9a023fe519c0fcbb0c5:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T10:42:22,422 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 7 files of size 251257 starting at candidate #0 after considering 15 permutations with 15 in ratio 2024-12-09T10:42:22,422 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1541): ec3eeaafd032b9a023fe519c0fcbb0c5/A is initiating minor compaction (all files) 2024-12-09T10:42:22,422 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ec3eeaafd032b9a023fe519c0fcbb0c5/A in TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. 2024-12-09T10:42:22,422 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/109a22e8b6744c7d82d4e0268b1f411c, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/6b2d64585f4c41dabb2b98b13a2c093a, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/865db025f7b64c9f8cc118aea566ff6e, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/ee5ffe6bb7b1459ca5355cd0dcdeb66b, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/79aa314b74184b4e8cd8009eb011b43b, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/b385f56d32984e1e930315290ecdc5d1, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/7fe5f09840f84199a2b78fe54d3c2347] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp, totalSize=245.4 K 2024-12-09T10:42:22,422 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=9 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. 2024-12-09T10:42:22,422 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. files: [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/109a22e8b6744c7d82d4e0268b1f411c, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/6b2d64585f4c41dabb2b98b13a2c093a, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/865db025f7b64c9f8cc118aea566ff6e, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/ee5ffe6bb7b1459ca5355cd0dcdeb66b, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/79aa314b74184b4e8cd8009eb011b43b, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/b385f56d32984e1e930315290ecdc5d1, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/7fe5f09840f84199a2b78fe54d3c2347] 2024-12-09T10:42:22,422 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 109a22e8b6744c7d82d4e0268b1f411c, keycount=300, bloomtype=ROW, size=55.4 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733740932252 2024-12-09T10:42:22,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:42:22,423 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b2d64585f4c41dabb2b98b13a2c093a, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733740932344 2024-12-09T10:42:22,423 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 7 store files, 0 compacting, 7 eligible, 16 blocking 2024-12-09T10:42:22,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ec3eeaafd032b9a023fe519c0fcbb0c5:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T10:42:22,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:42:22,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ec3eeaafd032b9a023fe519c0fcbb0c5:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T10:42:22,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T10:42:22,424 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 865db025f7b64c9f8cc118aea566ff6e, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1733740933476 2024-12-09T10:42:22,425 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting ee5ffe6bb7b1459ca5355cd0dcdeb66b, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1733740934644 2024-12-09T10:42:22,425 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 79aa314b74184b4e8cd8009eb011b43b, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1733740935714 2024-12-09T10:42:22,425 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting b385f56d32984e1e930315290ecdc5d1, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1733740938046 2024-12-09T10:42:22,426 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7fe5f09840f84199a2b78fe54d3c2347, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1733740940225 2024-12-09T10:42:22,427 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 7 files of size 84157 starting at candidate #0 after considering 15 permutations with 15 in ratio 2024-12-09T10:42:22,427 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1541): ec3eeaafd032b9a023fe519c0fcbb0c5/B is initiating minor compaction (all files) 2024-12-09T10:42:22,427 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ec3eeaafd032b9a023fe519c0fcbb0c5/B in TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. 2024-12-09T10:42:22,427 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B/5c0aa3f183f244bc9bf673c22c016d3a, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B/a9462ad893a142239d9222053b5908cd, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B/6b0fad94b2e94bb38296072812e15691, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B/e05a4d8e7b73445f97adea5aa4b23f9e, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B/fb883fa67fae4f72a07d754226b50f0f, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B/0bb5884d15904307ba4074f165fa274e, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B/e76673facc9e4d2a84856331c080b386] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp, totalSize=82.2 K 2024-12-09T10:42:22,428 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c0aa3f183f244bc9bf673c22c016d3a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733740932256 2024-12-09T10:42:22,428 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting a9462ad893a142239d9222053b5908cd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733740932344 2024-12-09T10:42:22,429 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b0fad94b2e94bb38296072812e15691, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1733740933476 2024-12-09T10:42:22,429 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting e05a4d8e7b73445f97adea5aa4b23f9e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1733740934644 2024-12-09T10:42:22,430 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting fb883fa67fae4f72a07d754226b50f0f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1733740935714 2024-12-09T10:42:22,431 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 0bb5884d15904307ba4074f165fa274e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1733740938046 2024-12-09T10:42:22,431 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting e76673facc9e4d2a84856331c080b386, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1733740940285 2024-12-09T10:42:22,446 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-09T10:42:22,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. 2024-12-09T10:42:22,446 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing ec3eeaafd032b9a023fe519c0fcbb0c5 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-09T10:42:22,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec3eeaafd032b9a023fe519c0fcbb0c5, store=A 2024-12-09T10:42:22,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:42:22,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec3eeaafd032b9a023fe519c0fcbb0c5, store=B 2024-12-09T10:42:22,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:42:22,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec3eeaafd032b9a023fe519c0fcbb0c5, store=C 2024-12-09T10:42:22,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:42:22,462 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ec3eeaafd032b9a023fe519c0fcbb0c5#B#compaction#306 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:42:22,463 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/B/05dc4a75a0444d44ad1ff5a8b5d3a642 is 50, key is test_row_0/B:col10/1733740940285/Put/seqid=0 2024-12-09T10:42:22,466 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ec3eeaafd032b9a023fe519c0fcbb0c5] 2024-12-09T10:42:22,498 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412095b4be27416ac45e6b3a356baf19c6c08_ec3eeaafd032b9a023fe519c0fcbb0c5 store=[table=TestAcidGuarantees family=A region=ec3eeaafd032b9a023fe519c0fcbb0c5] 2024-12-09T10:42:22,503 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412095b4be27416ac45e6b3a356baf19c6c08_ec3eeaafd032b9a023fe519c0fcbb0c5, store=[table=TestAcidGuarantees family=A region=ec3eeaafd032b9a023fe519c0fcbb0c5] 2024-12-09T10:42:22,503 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412095b4be27416ac45e6b3a356baf19c6c08_ec3eeaafd032b9a023fe519c0fcbb0c5 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ec3eeaafd032b9a023fe519c0fcbb0c5] 2024-12-09T10:42:22,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209c7929d69133f43c791344dbac64a1116_ec3eeaafd032b9a023fe519c0fcbb0c5 is 50, key is test_row_0/A:col10/1733740941516/Put/seqid=0 2024-12-09T10:42:22,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742185_1361 (size=12391) 2024-12-09T10:42:22,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742186_1362 (size=4469) 2024-12-09T10:42:22,553 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ec3eeaafd032b9a023fe519c0fcbb0c5#A#compaction#307 average throughput is 0.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:42:22,554 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/A/fff415a458144776ab8ba5d7c58d9ed9 is 175, key is test_row_0/A:col10/1733740940285/Put/seqid=0 2024-12-09T10:42:22,555 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/B/05dc4a75a0444d44ad1ff5a8b5d3a642 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B/05dc4a75a0444d44ad1ff5a8b5d3a642 2024-12-09T10:42:22,562 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 7 (all) file(s) in ec3eeaafd032b9a023fe519c0fcbb0c5/B of ec3eeaafd032b9a023fe519c0fcbb0c5 into 05dc4a75a0444d44ad1ff5a8b5d3a642(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:42:22,562 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ec3eeaafd032b9a023fe519c0fcbb0c5: 2024-12-09T10:42:22,562 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., storeName=ec3eeaafd032b9a023fe519c0fcbb0c5/B, priority=9, startTime=1733740942423; duration=0sec 2024-12-09T10:42:22,562 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T10:42:22,562 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ec3eeaafd032b9a023fe519c0fcbb0c5:B 2024-12-09T10:42:22,562 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 7 store files, 0 compacting, 7 eligible, 16 blocking 2024-12-09T10:42:22,570 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 7 files of size 84157 starting at candidate #0 after considering 15 permutations with 15 in ratio 2024-12-09T10:42:22,570 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1541): ec3eeaafd032b9a023fe519c0fcbb0c5/C is initiating minor compaction (all files) 2024-12-09T10:42:22,570 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ec3eeaafd032b9a023fe519c0fcbb0c5/C in TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. 2024-12-09T10:42:22,571 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C/20d65f890c4f46c19b993aabc4feda36, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C/162225dfd7c443eca45d98b39020b374, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C/a844161280444e78b808ed6353abf0e6, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C/8ec70b07680a4f5bad18ed997afd9d10, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C/431e8973d54d4052911a079d70c72e48, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C/ac17b63080d040a29c2825c823ce4af6, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C/f3bee8c13f544f2693a7e46e8bc888fd] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp, totalSize=82.2 K 2024-12-09T10:42:22,571 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 20d65f890c4f46c19b993aabc4feda36, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733740932256 2024-12-09T10:42:22,571 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 162225dfd7c443eca45d98b39020b374, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733740932344 2024-12-09T10:42:22,572 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting a844161280444e78b808ed6353abf0e6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1733740933476 2024-12-09T10:42:22,572 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ec70b07680a4f5bad18ed997afd9d10, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1733740934644 2024-12-09T10:42:22,572 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 431e8973d54d4052911a079d70c72e48, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1733740935714 2024-12-09T10:42:22,573 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting ac17b63080d040a29c2825c823ce4af6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1733740938046 2024-12-09T10:42:22,573 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting f3bee8c13f544f2693a7e46e8bc888fd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1733740940285 2024-12-09T10:42:22,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742187_1363 (size=12304) 2024-12-09T10:42:22,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:22,582 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209c7929d69133f43c791344dbac64a1116_ec3eeaafd032b9a023fe519c0fcbb0c5 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209c7929d69133f43c791344dbac64a1116_ec3eeaafd032b9a023fe519c0fcbb0c5 2024-12-09T10:42:22,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/A/d842be8dc62b4bad889d09b5176025c4, store: [table=TestAcidGuarantees family=A region=ec3eeaafd032b9a023fe519c0fcbb0c5] 2024-12-09T10:42:22,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/A/d842be8dc62b4bad889d09b5176025c4 is 175, key is test_row_0/A:col10/1733740941516/Put/seqid=0 2024-12-09T10:42:22,604 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ec3eeaafd032b9a023fe519c0fcbb0c5#C#compaction#309 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:42:22,605 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/C/cb5d0776a4ee47678246794670559d94 is 50, key is test_row_0/C:col10/1733740940285/Put/seqid=0 2024-12-09T10:42:22,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-09T10:42:22,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742188_1364 (size=31345) 2024-12-09T10:42:22,628 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/A/fff415a458144776ab8ba5d7c58d9ed9 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/fff415a458144776ab8ba5d7c58d9ed9 2024-12-09T10:42:22,633 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 7 (all) file(s) in ec3eeaafd032b9a023fe519c0fcbb0c5/A of ec3eeaafd032b9a023fe519c0fcbb0c5 into fff415a458144776ab8ba5d7c58d9ed9(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:42:22,633 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ec3eeaafd032b9a023fe519c0fcbb0c5: 2024-12-09T10:42:22,633 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., storeName=ec3eeaafd032b9a023fe519c0fcbb0c5/A, priority=9, startTime=1733740942419; duration=0sec 2024-12-09T10:42:22,633 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:42:22,633 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ec3eeaafd032b9a023fe519c0fcbb0c5:A 2024-12-09T10:42:22,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742189_1365 (size=31105) 2024-12-09T10:42:22,650 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=159, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/A/d842be8dc62b4bad889d09b5176025c4 2024-12-09T10:42:22,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742190_1366 (size=12391) 2024-12-09T10:42:22,671 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/C/cb5d0776a4ee47678246794670559d94 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C/cb5d0776a4ee47678246794670559d94 2024-12-09T10:42:22,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/B/5dac524d062d467c85b759e6f54d7e89 is 50, key is test_row_0/B:col10/1733740941516/Put/seqid=0 2024-12-09T10:42:22,685 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 7 (all) file(s) in ec3eeaafd032b9a023fe519c0fcbb0c5/C of ec3eeaafd032b9a023fe519c0fcbb0c5 into cb5d0776a4ee47678246794670559d94(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:42:22,685 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ec3eeaafd032b9a023fe519c0fcbb0c5: 2024-12-09T10:42:22,685 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., storeName=ec3eeaafd032b9a023fe519c0fcbb0c5/C, priority=9, startTime=1733740942423; duration=0sec 2024-12-09T10:42:22,685 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:42:22,685 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ec3eeaafd032b9a023fe519c0fcbb0c5:C 2024-12-09T10:42:22,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742191_1367 (size=12151) 2024-12-09T10:42:22,713 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/B/5dac524d062d467c85b759e6f54d7e89 2024-12-09T10:42:22,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(8855): Flush requested on ec3eeaafd032b9a023fe519c0fcbb0c5 2024-12-09T10:42:22,732 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. as already flushing 2024-12-09T10:42:22,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/C/b6f9ee9ca9a64f2b9e6c83b4414bffcc is 50, key is test_row_0/C:col10/1733740941516/Put/seqid=0 2024-12-09T10:42:22,780 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:22,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50954 deadline: 1733741002762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:22,781 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:22,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50962 deadline: 1733741002764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:22,781 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:22,781 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:22,781 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:22,781 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:22,781 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:22,781 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:22,794 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:22,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50928 deadline: 1733741002781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:22,795 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:22,795 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:22,795 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:22,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742192_1368 (size=12151) 2024-12-09T10:42:22,810 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/C/b6f9ee9ca9a64f2b9e6c83b4414bffcc 2024-12-09T10:42:22,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/A/d842be8dc62b4bad889d09b5176025c4 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/d842be8dc62b4bad889d09b5176025c4 2024-12-09T10:42:22,824 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/d842be8dc62b4bad889d09b5176025c4, entries=150, sequenceid=159, filesize=30.4 K 2024-12-09T10:42:22,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/B/5dac524d062d467c85b759e6f54d7e89 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B/5dac524d062d467c85b759e6f54d7e89 2024-12-09T10:42:22,831 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B/5dac524d062d467c85b759e6f54d7e89, entries=150, sequenceid=159, filesize=11.9 K 2024-12-09T10:42:22,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/C/b6f9ee9ca9a64f2b9e6c83b4414bffcc as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C/b6f9ee9ca9a64f2b9e6c83b4414bffcc 2024-12-09T10:42:22,844 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C/b6f9ee9ca9a64f2b9e6c83b4414bffcc, entries=150, sequenceid=159, filesize=11.9 K 2024-12-09T10:42:22,845 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for ec3eeaafd032b9a023fe519c0fcbb0c5 in 399ms, sequenceid=159, compaction requested=false 2024-12-09T10:42:22,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for ec3eeaafd032b9a023fe519c0fcbb0c5: 2024-12-09T10:42:22,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. 2024-12-09T10:42:22,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-12-09T10:42:22,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-12-09T10:42:22,850 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-12-09T10:42:22,850 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 863 msec 2024-12-09T10:42:22,852 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 870 msec 2024-12-09T10:42:22,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(8855): Flush requested on ec3eeaafd032b9a023fe519c0fcbb0c5 2024-12-09T10:42:22,891 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ec3eeaafd032b9a023fe519c0fcbb0c5 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-09T10:42:22,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec3eeaafd032b9a023fe519c0fcbb0c5, store=A 2024-12-09T10:42:22,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:42:22,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec3eeaafd032b9a023fe519c0fcbb0c5, store=B 2024-12-09T10:42:22,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:42:22,892 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec3eeaafd032b9a023fe519c0fcbb0c5, store=C 2024-12-09T10:42:22,892 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:42:22,904 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412097b2c82030b3748e8a16895b22263069b_ec3eeaafd032b9a023fe519c0fcbb0c5 is 50, key is test_row_0/A:col10/1733740942762/Put/seqid=0 2024-12-09T10:42:22,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742193_1369 (size=14794) 2024-12-09T10:42:22,913 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:22,917 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412097b2c82030b3748e8a16895b22263069b_ec3eeaafd032b9a023fe519c0fcbb0c5 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412097b2c82030b3748e8a16895b22263069b_ec3eeaafd032b9a023fe519c0fcbb0c5 2024-12-09T10:42:22,918 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/A/8f147ed875d842ed955e986f85ff670b, store: [table=TestAcidGuarantees family=A region=ec3eeaafd032b9a023fe519c0fcbb0c5] 2024-12-09T10:42:22,918 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/A/8f147ed875d842ed955e986f85ff670b is 175, key is test_row_0/A:col10/1733740942762/Put/seqid=0 2024-12-09T10:42:22,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742194_1370 (size=39749) 2024-12-09T10:42:22,928 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=176, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/A/8f147ed875d842ed955e986f85ff670b 2024-12-09T10:42:22,936 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/B/9b6bec719f7c483a86b583da919ea1b9 is 50, key is test_row_0/B:col10/1733740942762/Put/seqid=0 2024-12-09T10:42:22,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742195_1371 (size=12151) 2024-12-09T10:42:22,944 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:22,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50928 deadline: 1733741002929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:22,944 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:22,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50942 deadline: 1733741002930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:22,944 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:22,944 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:22,944 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:22,944 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:22,945 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:22,945 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:22,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50962 deadline: 1733741002931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:22,945 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:22,945 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:22,945 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:22,945 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:22,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:22,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50974 deadline: 1733741002938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:22,948 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:22,948 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:22,948 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:22,954 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:22,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50954 deadline: 1733741002939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:22,954 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:22,954 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:22,954 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:23,055 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:23,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50928 deadline: 1733741003049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:23,055 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:23,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50942 deadline: 1733741003050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:23,055 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:23,055 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:23,055 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:23,055 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:23,055 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:23,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50962 deadline: 1733741003050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:23,056 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:23,056 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:23,056 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:23,056 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:23,056 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:23,070 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:23,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50974 deadline: 1733741003059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:23,070 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:23,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50954 deadline: 1733741003059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:23,070 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:23,071 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:23,071 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:23,071 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:23,071 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:23,071 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:23,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-09T10:42:23,120 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T10:42:23,121 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-09T10:42:23,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-12-09T10:42:23,123 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T10:42:23,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-09T10:42:23,123 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T10:42:23,124 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T10:42:23,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-09T10:42:23,268 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:23,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50928 deadline: 1733741003260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:23,268 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:23,269 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:23,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50942 deadline: 1733741003260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:23,269 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:23,269 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:23,269 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:23,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50962 deadline: 1733741003260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:23,274 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:23,274 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:23,274 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:23,274 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:23,274 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:23,274 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:23,279 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-09T10:42:23,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. 2024-12-09T10:42:23,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. as already flushing 2024-12-09T10:42:23,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. 2024-12-09T10:42:23,279 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => ec3eeaafd032b9a023fe519c0fcbb0c5, NAME => 'TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:23,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => ec3eeaafd032b9a023fe519c0fcbb0c5, NAME => 'TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:23,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ec3eeaafd032b9a023fe519c0fcbb0c5, NAME => 'TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ec3eeaafd032b9a023fe519c0fcbb0c5, NAME => 'TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:23,294 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:23,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50954 deadline: 1733741003280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:23,295 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:23,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50974 deadline: 1733741003280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:23,295 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:23,295 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:23,295 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:23,295 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:23,296 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:23,296 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:23,342 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/B/9b6bec719f7c483a86b583da919ea1b9 2024-12-09T10:42:23,370 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/C/a4ee336c554c4e849c0aed23c7e0ce63 is 50, key is test_row_0/C:col10/1733740942762/Put/seqid=0 2024-12-09T10:42:23,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742196_1372 (size=12151) 2024-12-09T10:42:23,426 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/C/a4ee336c554c4e849c0aed23c7e0ce63 2024-12-09T10:42:23,431 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-09T10:42:23,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. 2024-12-09T10:42:23,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. as already flushing 2024-12-09T10:42:23,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. 2024-12-09T10:42:23,432 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => ec3eeaafd032b9a023fe519c0fcbb0c5, NAME => 'TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:23,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => ec3eeaafd032b9a023fe519c0fcbb0c5, NAME => 'TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:23,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ec3eeaafd032b9a023fe519c0fcbb0c5, NAME => 'TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ec3eeaafd032b9a023fe519c0fcbb0c5, NAME => 'TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:23,436 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/A/8f147ed875d842ed955e986f85ff670b as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/8f147ed875d842ed955e986f85ff670b 2024-12-09T10:42:23,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-09T10:42:23,440 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/8f147ed875d842ed955e986f85ff670b, entries=200, sequenceid=176, filesize=38.8 K 2024-12-09T10:42:23,441 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/B/9b6bec719f7c483a86b583da919ea1b9 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B/9b6bec719f7c483a86b583da919ea1b9 2024-12-09T10:42:23,446 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B/9b6bec719f7c483a86b583da919ea1b9, entries=150, sequenceid=176, filesize=11.9 K 2024-12-09T10:42:23,447 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/C/a4ee336c554c4e849c0aed23c7e0ce63 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C/a4ee336c554c4e849c0aed23c7e0ce63 2024-12-09T10:42:23,452 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C/a4ee336c554c4e849c0aed23c7e0ce63, entries=150, sequenceid=176, filesize=11.9 K 2024-12-09T10:42:23,454 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for ec3eeaafd032b9a023fe519c0fcbb0c5 in 563ms, sequenceid=176, compaction requested=true 2024-12-09T10:42:23,454 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ec3eeaafd032b9a023fe519c0fcbb0c5: 2024-12-09T10:42:23,454 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ec3eeaafd032b9a023fe519c0fcbb0c5:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T10:42:23,454 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:42:23,454 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T10:42:23,454 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ec3eeaafd032b9a023fe519c0fcbb0c5:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T10:42:23,454 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T10:42:23,454 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ec3eeaafd032b9a023fe519c0fcbb0c5:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T10:42:23,454 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-09T10:42:23,454 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T10:42:23,457 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T10:42:23,457 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1541): ec3eeaafd032b9a023fe519c0fcbb0c5/B is initiating minor compaction (all files) 2024-12-09T10:42:23,457 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ec3eeaafd032b9a023fe519c0fcbb0c5/B in TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. 2024-12-09T10:42:23,457 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B/05dc4a75a0444d44ad1ff5a8b5d3a642, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B/5dac524d062d467c85b759e6f54d7e89, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B/9b6bec719f7c483a86b583da919ea1b9] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp, totalSize=35.8 K 2024-12-09T10:42:23,457 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102199 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T10:42:23,457 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1541): ec3eeaafd032b9a023fe519c0fcbb0c5/A is initiating minor compaction (all files) 2024-12-09T10:42:23,458 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ec3eeaafd032b9a023fe519c0fcbb0c5/A in TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. 2024-12-09T10:42:23,458 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/fff415a458144776ab8ba5d7c58d9ed9, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/d842be8dc62b4bad889d09b5176025c4, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/8f147ed875d842ed955e986f85ff670b] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp, totalSize=99.8 K 2024-12-09T10:42:23,458 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 05dc4a75a0444d44ad1ff5a8b5d3a642, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1733740940285 2024-12-09T10:42:23,458 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. 2024-12-09T10:42:23,458 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. files: [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/fff415a458144776ab8ba5d7c58d9ed9, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/d842be8dc62b4bad889d09b5176025c4, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/8f147ed875d842ed955e986f85ff670b] 2024-12-09T10:42:23,458 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 5dac524d062d467c85b759e6f54d7e89, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1733740941510 2024-12-09T10:42:23,458 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting fff415a458144776ab8ba5d7c58d9ed9, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1733740940285 2024-12-09T10:42:23,459 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b6bec719f7c483a86b583da919ea1b9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1733740942762 2024-12-09T10:42:23,459 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting d842be8dc62b4bad889d09b5176025c4, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1733740941510 2024-12-09T10:42:23,459 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8f147ed875d842ed955e986f85ff670b, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1733740942760 2024-12-09T10:42:23,482 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ec3eeaafd032b9a023fe519c0fcbb0c5#B#compaction#315 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:42:23,482 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/B/01bee7bdac78474fbeace6788f12ddb3 is 50, key is test_row_0/B:col10/1733740942762/Put/seqid=0 2024-12-09T10:42:23,498 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ec3eeaafd032b9a023fe519c0fcbb0c5] 2024-12-09T10:42:23,518 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120975728f0f5b6b4c369813ad2e232c5404_ec3eeaafd032b9a023fe519c0fcbb0c5 store=[table=TestAcidGuarantees family=A region=ec3eeaafd032b9a023fe519c0fcbb0c5] 2024-12-09T10:42:23,520 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120975728f0f5b6b4c369813ad2e232c5404_ec3eeaafd032b9a023fe519c0fcbb0c5, store=[table=TestAcidGuarantees family=A region=ec3eeaafd032b9a023fe519c0fcbb0c5] 2024-12-09T10:42:23,520 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120975728f0f5b6b4c369813ad2e232c5404_ec3eeaafd032b9a023fe519c0fcbb0c5 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ec3eeaafd032b9a023fe519c0fcbb0c5] 2024-12-09T10:42:23,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742197_1373 (size=12493) 2024-12-09T10:42:23,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742198_1374 (size=4469) 2024-12-09T10:42:23,586 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-09T10:42:23,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. 2024-12-09T10:42:23,586 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing ec3eeaafd032b9a023fe519c0fcbb0c5 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-09T10:42:23,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec3eeaafd032b9a023fe519c0fcbb0c5, store=A 2024-12-09T10:42:23,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:42:23,587 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ec3eeaafd032b9a023fe519c0fcbb0c5#A#compaction#316 average throughput is 0.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:42:23,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec3eeaafd032b9a023fe519c0fcbb0c5, store=B 2024-12-09T10:42:23,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:42:23,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec3eeaafd032b9a023fe519c0fcbb0c5, store=C 2024-12-09T10:42:23,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:42:23,587 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/A/51d4cbed64d44502b1bf112f90605b46 is 175, key is test_row_0/A:col10/1733740942762/Put/seqid=0 2024-12-09T10:42:23,591 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. as already flushing 2024-12-09T10:42:23,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(8855): Flush requested on ec3eeaafd032b9a023fe519c0fcbb0c5 2024-12-09T10:42:23,637 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:23,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50954 deadline: 1733741003615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:23,637 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:23,637 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:23,637 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:23,637 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:23,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50974 deadline: 1733741003616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:23,638 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:23,638 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:23,638 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:23,643 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:23,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50942 deadline: 1733741003629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:23,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:23,644 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:23,644 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:23,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209b8fc284b36ec421ba3bd323e296424ac_ec3eeaafd032b9a023fe519c0fcbb0c5 is 50, key is test_row_0/A:col10/1733740942937/Put/seqid=0 2024-12-09T10:42:23,656 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:23,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50928 deadline: 1733741003635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:23,656 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:23,656 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:23,657 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:23,659 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:23,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50962 deadline: 1733741003637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:23,659 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:23,659 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:23,659 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:23,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742199_1375 (size=31447) 2024-12-09T10:42:23,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742200_1376 (size=12304) 2024-12-09T10:42:23,681 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/A/51d4cbed64d44502b1bf112f90605b46 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/51d4cbed64d44502b1bf112f90605b46 2024-12-09T10:42:23,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:23,693 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209b8fc284b36ec421ba3bd323e296424ac_ec3eeaafd032b9a023fe519c0fcbb0c5 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209b8fc284b36ec421ba3bd323e296424ac_ec3eeaafd032b9a023fe519c0fcbb0c5 2024-12-09T10:42:23,700 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ec3eeaafd032b9a023fe519c0fcbb0c5/A of ec3eeaafd032b9a023fe519c0fcbb0c5 into 51d4cbed64d44502b1bf112f90605b46(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:42:23,700 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ec3eeaafd032b9a023fe519c0fcbb0c5: 2024-12-09T10:42:23,700 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., storeName=ec3eeaafd032b9a023fe519c0fcbb0c5/A, priority=13, startTime=1733740943454; duration=0sec 2024-12-09T10:42:23,700 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T10:42:23,700 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ec3eeaafd032b9a023fe519c0fcbb0c5:A 2024-12-09T10:42:23,700 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T10:42:23,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/A/26d5c24373ad4374b4a561f52633c68e, store: [table=TestAcidGuarantees family=A region=ec3eeaafd032b9a023fe519c0fcbb0c5] 2024-12-09T10:42:23,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/A/26d5c24373ad4374b4a561f52633c68e is 175, key is test_row_0/A:col10/1733740942937/Put/seqid=0 2024-12-09T10:42:23,708 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T10:42:23,708 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1541): ec3eeaafd032b9a023fe519c0fcbb0c5/C is initiating minor compaction (all files) 2024-12-09T10:42:23,709 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ec3eeaafd032b9a023fe519c0fcbb0c5/C in TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. 2024-12-09T10:42:23,709 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C/cb5d0776a4ee47678246794670559d94, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C/b6f9ee9ca9a64f2b9e6c83b4414bffcc, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C/a4ee336c554c4e849c0aed23c7e0ce63] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp, totalSize=35.8 K 2024-12-09T10:42:23,710 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting cb5d0776a4ee47678246794670559d94, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1733740940285 2024-12-09T10:42:23,710 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting b6f9ee9ca9a64f2b9e6c83b4414bffcc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1733740941510 2024-12-09T10:42:23,710 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting a4ee336c554c4e849c0aed23c7e0ce63, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1733740942762 2024-12-09T10:42:23,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-09T10:42:23,753 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ec3eeaafd032b9a023fe519c0fcbb0c5#C#compaction#318 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:42:23,753 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/C/223a66ea80134e0f92de8ebaff18ef51 is 50, key is test_row_0/C:col10/1733740942762/Put/seqid=0 2024-12-09T10:42:23,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742201_1377 (size=31105) 2024-12-09T10:42:23,760 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=194, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/A/26d5c24373ad4374b4a561f52633c68e 2024-12-09T10:42:23,764 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:23,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50954 deadline: 1733741003739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:23,765 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:23,765 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:23,765 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:23,774 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:23,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50974 deadline: 1733741003750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:23,775 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:23,775 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:23,775 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:23,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50942 deadline: 1733741003750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:23,775 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:23,775 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:23,776 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:23,776 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:23,776 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:23,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50928 deadline: 1733741003759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:23,776 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:23,776 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:23,776 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:23,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/B/5b1869c5cf294980bba672cd66b3b5a9 is 50, key is test_row_0/B:col10/1733740942937/Put/seqid=0 2024-12-09T10:42:23,788 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:23,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50962 deadline: 1733741003770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:23,789 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:23,789 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:23,789 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:23,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742202_1378 (size=12493) 2024-12-09T10:42:23,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742203_1379 (size=12151) 2024-12-09T10:42:23,951 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/B/01bee7bdac78474fbeace6788f12ddb3 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B/01bee7bdac78474fbeace6788f12ddb3 2024-12-09T10:42:23,958 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ec3eeaafd032b9a023fe519c0fcbb0c5/B of ec3eeaafd032b9a023fe519c0fcbb0c5 into 01bee7bdac78474fbeace6788f12ddb3(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:42:23,958 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ec3eeaafd032b9a023fe519c0fcbb0c5: 2024-12-09T10:42:23,958 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., storeName=ec3eeaafd032b9a023fe519c0fcbb0c5/B, priority=13, startTime=1733740943454; duration=0sec 2024-12-09T10:42:23,958 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:42:23,958 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ec3eeaafd032b9a023fe519c0fcbb0c5:B 2024-12-09T10:42:23,988 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:23,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50954 deadline: 1733741003969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:23,989 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:23,989 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:23,989 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:23,989 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:23,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50974 deadline: 1733741003980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:23,990 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:23,990 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:23,990 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:23,990 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:23,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50942 deadline: 1733741003980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:23,990 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:23,990 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:23,990 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:23,990 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:23,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50928 deadline: 1733741003980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:23,991 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:23,991 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:23,991 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:24,012 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:24,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50962 deadline: 1733741003999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:24,013 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:24,013 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:24,013 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:24,209 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/C/223a66ea80134e0f92de8ebaff18ef51 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C/223a66ea80134e0f92de8ebaff18ef51 2024-12-09T10:42:24,213 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ec3eeaafd032b9a023fe519c0fcbb0c5/C of ec3eeaafd032b9a023fe519c0fcbb0c5 into 223a66ea80134e0f92de8ebaff18ef51(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:42:24,213 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ec3eeaafd032b9a023fe519c0fcbb0c5: 2024-12-09T10:42:24,213 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., storeName=ec3eeaafd032b9a023fe519c0fcbb0c5/C, priority=13, startTime=1733740943454; duration=0sec 2024-12-09T10:42:24,214 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:42:24,214 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ec3eeaafd032b9a023fe519c0fcbb0c5:C 2024-12-09T10:42:24,240 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/B/5b1869c5cf294980bba672cd66b3b5a9 2024-12-09T10:42:24,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-09T10:42:24,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/C/1b126eba30ac4329a300afaa3fe08ce3 is 50, key is test_row_0/C:col10/1733740942937/Put/seqid=0 2024-12-09T10:42:24,315 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:24,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50942 deadline: 1733741004300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:24,315 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:24,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50974 deadline: 1733741004300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:24,316 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:24,316 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:24,316 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:24,316 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:24,316 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:24,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50954 deadline: 1733741004300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:24,316 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:24,316 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:24,316 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:24,316 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:24,316 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:24,316 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:24,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50928 deadline: 1733741004300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:24,317 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:24,317 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:24,317 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:24,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742204_1380 (size=12151) 2024-12-09T10:42:24,334 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:24,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50962 deadline: 1733741004319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:24,335 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:24,335 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:24,335 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:24,727 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/C/1b126eba30ac4329a300afaa3fe08ce3 2024-12-09T10:42:24,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/A/26d5c24373ad4374b4a561f52633c68e as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/26d5c24373ad4374b4a561f52633c68e 2024-12-09T10:42:24,737 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/26d5c24373ad4374b4a561f52633c68e, entries=150, sequenceid=194, filesize=30.4 K 2024-12-09T10:42:24,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/B/5b1869c5cf294980bba672cd66b3b5a9 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B/5b1869c5cf294980bba672cd66b3b5a9 2024-12-09T10:42:24,743 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B/5b1869c5cf294980bba672cd66b3b5a9, entries=150, sequenceid=194, filesize=11.9 K 2024-12-09T10:42:24,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/C/1b126eba30ac4329a300afaa3fe08ce3 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C/1b126eba30ac4329a300afaa3fe08ce3 2024-12-09T10:42:24,748 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C/1b126eba30ac4329a300afaa3fe08ce3, entries=150, sequenceid=194, filesize=11.9 K 2024-12-09T10:42:24,749 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for ec3eeaafd032b9a023fe519c0fcbb0c5 in 1163ms, sequenceid=194, compaction requested=false 2024-12-09T10:42:24,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for ec3eeaafd032b9a023fe519c0fcbb0c5: 2024-12-09T10:42:24,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. 2024-12-09T10:42:24,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-12-09T10:42:24,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-12-09T10:42:24,754 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-12-09T10:42:24,754 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6270 sec 2024-12-09T10:42:24,756 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 1.6340 sec 2024-12-09T10:42:24,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(8855): Flush requested on ec3eeaafd032b9a023fe519c0fcbb0c5 2024-12-09T10:42:24,828 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ec3eeaafd032b9a023fe519c0fcbb0c5 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-09T10:42:24,828 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec3eeaafd032b9a023fe519c0fcbb0c5, store=A 2024-12-09T10:42:24,828 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:42:24,828 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec3eeaafd032b9a023fe519c0fcbb0c5, store=B 2024-12-09T10:42:24,828 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:42:24,828 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec3eeaafd032b9a023fe519c0fcbb0c5, store=C 2024-12-09T10:42:24,828 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:42:24,838 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209bdc627d5ab7c4749bc3d2ff9af0b7f6c_ec3eeaafd032b9a023fe519c0fcbb0c5 is 50, key is test_row_0/A:col10/1733740943614/Put/seqid=0 2024-12-09T10:42:24,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742205_1381 (size=14794) 2024-12-09T10:42:24,851 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:24,856 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209bdc627d5ab7c4749bc3d2ff9af0b7f6c_ec3eeaafd032b9a023fe519c0fcbb0c5 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209bdc627d5ab7c4749bc3d2ff9af0b7f6c_ec3eeaafd032b9a023fe519c0fcbb0c5 2024-12-09T10:42:24,858 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/A/3344d67626d84d8193d8b75dafc6b296, store: [table=TestAcidGuarantees family=A region=ec3eeaafd032b9a023fe519c0fcbb0c5] 2024-12-09T10:42:24,858 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/A/3344d67626d84d8193d8b75dafc6b296 is 175, key is test_row_0/A:col10/1733740943614/Put/seqid=0 2024-12-09T10:42:24,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742206_1382 (size=39749) 2024-12-09T10:42:24,890 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:24,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50942 deadline: 1733741004866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:24,891 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:24,891 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:24,891 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:24,903 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:24,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50974 deadline: 1733741004880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:24,904 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:24,904 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:24,904 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:24,908 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:24,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50954 deadline: 1733741004891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:24,909 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:24,909 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:24,908 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:24,909 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:24,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50962 deadline: 1733741004891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:24,909 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:24,909 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:24,909 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:24,909 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:24,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50928 deadline: 1733741004891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:24,909 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:24,909 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:24,909 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:25,011 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:25,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50942 deadline: 1733741004999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:25,011 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:25,011 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:25,011 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:25,014 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:25,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50974 deadline: 1733741005010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:25,014 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:25,014 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:25,014 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:25,027 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:25,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50954 deadline: 1733741005019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:25,027 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:25,027 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:25,027 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:25,027 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:25,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50962 deadline: 1733741005019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:25,028 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:25,028 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:25,028 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:25,028 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:25,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50928 deadline: 1733741005019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:25,028 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:25,028 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:25,028 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:25,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:25,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50942 deadline: 1733741005219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:25,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:25,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50974 deadline: 1733741005219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:25,229 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:25,229 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:25,229 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:25,230 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:25,230 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:25,230 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:25,237 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:25,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50954 deadline: 1733741005229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:25,237 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:25,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50928 deadline: 1733741005230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:25,237 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:25,237 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:25,237 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:25,237 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:25,238 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:25,238 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:25,250 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:25,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50962 deadline: 1733741005240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:25,250 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:25,250 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:25,250 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:25,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-09T10:42:25,269 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T10:42:25,270 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-09T10:42:25,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-12-09T10:42:25,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-09T10:42:25,271 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T10:42:25,272 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T10:42:25,272 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T10:42:25,290 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=212, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/A/3344d67626d84d8193d8b75dafc6b296 2024-12-09T10:42:25,306 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/B/a50f00f7126144e285937ccb895062fd is 50, key is test_row_0/B:col10/1733740943614/Put/seqid=0 2024-12-09T10:42:25,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742207_1383 (size=12151) 2024-12-09T10:42:25,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-09T10:42:25,424 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-09T10:42:25,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. 2024-12-09T10:42:25,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. as already flushing 2024-12-09T10:42:25,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. 2024-12-09T10:42:25,425 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => ec3eeaafd032b9a023fe519c0fcbb0c5, NAME => 'TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:25,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => ec3eeaafd032b9a023fe519c0fcbb0c5, NAME => 'TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:25,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ec3eeaafd032b9a023fe519c0fcbb0c5, NAME => 'TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ec3eeaafd032b9a023fe519c0fcbb0c5, NAME => 'TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:25,547 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:25,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50942 deadline: 1733741005539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:25,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:25,548 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:25,548 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:25,552 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:25,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50974 deadline: 1733741005540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:25,552 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:25,552 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:25,552 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:25,560 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:25,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50954 deadline: 1733741005550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:25,561 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:25,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50928 deadline: 1733741005550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:25,561 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:25,561 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:25,561 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:25,561 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:25,561 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:25,561 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:25,568 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:25,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50962 deadline: 1733741005559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:25,569 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:25,569 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:25,569 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:25,577 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-09T10:42:25,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. 2024-12-09T10:42:25,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. as already flushing 2024-12-09T10:42:25,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. 2024-12-09T10:42:25,578 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => ec3eeaafd032b9a023fe519c0fcbb0c5, NAME => 'TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:25,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => ec3eeaafd032b9a023fe519c0fcbb0c5, NAME => 'TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:25,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ec3eeaafd032b9a023fe519c0fcbb0c5, NAME => 'TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ec3eeaafd032b9a023fe519c0fcbb0c5, NAME => 'TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:25,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-09T10:42:25,712 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/B/a50f00f7126144e285937ccb895062fd 2024-12-09T10:42:25,719 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/C/703c5e801b6c45d89cd46f0fd9d89740 is 50, key is test_row_0/C:col10/1733740943614/Put/seqid=0 2024-12-09T10:42:25,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742208_1384 (size=12151) 2024-12-09T10:42:25,731 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-09T10:42:25,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. 2024-12-09T10:42:25,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. as already flushing 2024-12-09T10:42:25,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. 2024-12-09T10:42:25,731 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => ec3eeaafd032b9a023fe519c0fcbb0c5, NAME => 'TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:25,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => ec3eeaafd032b9a023fe519c0fcbb0c5, NAME => 'TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:25,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ec3eeaafd032b9a023fe519c0fcbb0c5, NAME => 'TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ec3eeaafd032b9a023fe519c0fcbb0c5, NAME => 'TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:25,885 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-09T10:42:25,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. 2024-12-09T10:42:25,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. as already flushing 2024-12-09T10:42:25,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. 2024-12-09T10:42:25,886 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => ec3eeaafd032b9a023fe519c0fcbb0c5, NAME => 'TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:25,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => ec3eeaafd032b9a023fe519c0fcbb0c5, NAME => 'TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:25,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ec3eeaafd032b9a023fe519c0fcbb0c5, NAME => 'TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ec3eeaafd032b9a023fe519c0fcbb0c5, NAME => 'TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:25,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-09T10:42:25,933 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-09T10:42:26,039 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-09T10:42:26,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. 2024-12-09T10:42:26,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. as already flushing 2024-12-09T10:42:26,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. 2024-12-09T10:42:26,039 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => ec3eeaafd032b9a023fe519c0fcbb0c5, NAME => 'TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:26,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => ec3eeaafd032b9a023fe519c0fcbb0c5, NAME => 'TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:26,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ec3eeaafd032b9a023fe519c0fcbb0c5, NAME => 'TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ec3eeaafd032b9a023fe519c0fcbb0c5, NAME => 'TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:26,076 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:26,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50942 deadline: 1733741006059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:26,076 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:26,076 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:26,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50974 deadline: 1733741006059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:26,077 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:26,077 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:26,077 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:26,077 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:26,077 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:26,085 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:26,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50954 deadline: 1733741006075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:26,086 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:26,086 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:26,086 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:26,086 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:26,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50928 deadline: 1733741006075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:26,086 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:26,086 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:26,086 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:26,088 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:26,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50962 deadline: 1733741006079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:26,089 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:26,089 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:26,089 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:26,124 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/C/703c5e801b6c45d89cd46f0fd9d89740 2024-12-09T10:42:26,129 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/A/3344d67626d84d8193d8b75dafc6b296 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/3344d67626d84d8193d8b75dafc6b296 2024-12-09T10:42:26,132 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/3344d67626d84d8193d8b75dafc6b296, entries=200, sequenceid=212, filesize=38.8 K 2024-12-09T10:42:26,133 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/B/a50f00f7126144e285937ccb895062fd as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B/a50f00f7126144e285937ccb895062fd 2024-12-09T10:42:26,136 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B/a50f00f7126144e285937ccb895062fd, entries=150, sequenceid=212, filesize=11.9 K 2024-12-09T10:42:26,137 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/C/703c5e801b6c45d89cd46f0fd9d89740 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C/703c5e801b6c45d89cd46f0fd9d89740 2024-12-09T10:42:26,140 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C/703c5e801b6c45d89cd46f0fd9d89740, entries=150, sequenceid=212, filesize=11.9 K 2024-12-09T10:42:26,144 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for ec3eeaafd032b9a023fe519c0fcbb0c5 in 1316ms, sequenceid=212, compaction requested=true 2024-12-09T10:42:26,144 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ec3eeaafd032b9a023fe519c0fcbb0c5: 2024-12-09T10:42:26,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ec3eeaafd032b9a023fe519c0fcbb0c5:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T10:42:26,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:42:26,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ec3eeaafd032b9a023fe519c0fcbb0c5:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T10:42:26,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:42:26,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ec3eeaafd032b9a023fe519c0fcbb0c5:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T10:42:26,144 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T10:42:26,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T10:42:26,144 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T10:42:26,145 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102301 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T10:42:26,145 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1541): ec3eeaafd032b9a023fe519c0fcbb0c5/A is initiating minor compaction (all files) 2024-12-09T10:42:26,145 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T10:42:26,145 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ec3eeaafd032b9a023fe519c0fcbb0c5/A in TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. 2024-12-09T10:42:26,145 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1541): ec3eeaafd032b9a023fe519c0fcbb0c5/B is initiating minor compaction (all files) 2024-12-09T10:42:26,145 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ec3eeaafd032b9a023fe519c0fcbb0c5/B in TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. 2024-12-09T10:42:26,145 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/51d4cbed64d44502b1bf112f90605b46, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/26d5c24373ad4374b4a561f52633c68e, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/3344d67626d84d8193d8b75dafc6b296] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp, totalSize=99.9 K 2024-12-09T10:42:26,145 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. 2024-12-09T10:42:26,145 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B/01bee7bdac78474fbeace6788f12ddb3, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B/5b1869c5cf294980bba672cd66b3b5a9, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B/a50f00f7126144e285937ccb895062fd] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp, totalSize=35.9 K 2024-12-09T10:42:26,145 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. files: [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/51d4cbed64d44502b1bf112f90605b46, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/26d5c24373ad4374b4a561f52633c68e, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/3344d67626d84d8193d8b75dafc6b296] 2024-12-09T10:42:26,146 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 51d4cbed64d44502b1bf112f90605b46, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1733740942762 2024-12-09T10:42:26,146 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 01bee7bdac78474fbeace6788f12ddb3, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1733740942762 2024-12-09T10:42:26,146 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 26d5c24373ad4374b4a561f52633c68e, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1733740942913 2024-12-09T10:42:26,146 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b1869c5cf294980bba672cd66b3b5a9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1733740942913 2024-12-09T10:42:26,146 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting a50f00f7126144e285937ccb895062fd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1733740943614 2024-12-09T10:42:26,146 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3344d67626d84d8193d8b75dafc6b296, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1733740943614 2024-12-09T10:42:26,154 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ec3eeaafd032b9a023fe519c0fcbb0c5] 2024-12-09T10:42:26,156 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ec3eeaafd032b9a023fe519c0fcbb0c5#B#compaction#324 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:42:26,156 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412099834b201313a4066887a98a08bc49e43_ec3eeaafd032b9a023fe519c0fcbb0c5 store=[table=TestAcidGuarantees family=A region=ec3eeaafd032b9a023fe519c0fcbb0c5] 2024-12-09T10:42:26,156 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/B/c35118f7f47b4a2f9d687bf80e6943df is 50, key is test_row_0/B:col10/1733740943614/Put/seqid=0 2024-12-09T10:42:26,158 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412099834b201313a4066887a98a08bc49e43_ec3eeaafd032b9a023fe519c0fcbb0c5, store=[table=TestAcidGuarantees family=A region=ec3eeaafd032b9a023fe519c0fcbb0c5] 2024-12-09T10:42:26,158 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412099834b201313a4066887a98a08bc49e43_ec3eeaafd032b9a023fe519c0fcbb0c5 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ec3eeaafd032b9a023fe519c0fcbb0c5] 2024-12-09T10:42:26,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742209_1385 (size=12595) 2024-12-09T10:42:26,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742210_1386 (size=4469) 2024-12-09T10:42:26,192 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-09T10:42:26,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. 2024-12-09T10:42:26,192 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing ec3eeaafd032b9a023fe519c0fcbb0c5 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-09T10:42:26,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec3eeaafd032b9a023fe519c0fcbb0c5, store=A 2024-12-09T10:42:26,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:42:26,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec3eeaafd032b9a023fe519c0fcbb0c5, store=B 2024-12-09T10:42:26,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:42:26,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec3eeaafd032b9a023fe519c0fcbb0c5, store=C 2024-12-09T10:42:26,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:42:26,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412091b691da6f0b64aed9752e6dd681b62c2_ec3eeaafd032b9a023fe519c0fcbb0c5 is 50, key is test_row_0/A:col10/1733740944859/Put/seqid=0 2024-12-09T10:42:26,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742211_1387 (size=12304) 2024-12-09T10:42:26,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:26,214 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412091b691da6f0b64aed9752e6dd681b62c2_ec3eeaafd032b9a023fe519c0fcbb0c5 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412091b691da6f0b64aed9752e6dd681b62c2_ec3eeaafd032b9a023fe519c0fcbb0c5 2024-12-09T10:42:26,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/A/ee5e60d6a20c47e49656a630a45c45b2, store: [table=TestAcidGuarantees family=A region=ec3eeaafd032b9a023fe519c0fcbb0c5] 2024-12-09T10:42:26,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/A/ee5e60d6a20c47e49656a630a45c45b2 is 175, key is test_row_0/A:col10/1733740944859/Put/seqid=0 2024-12-09T10:42:26,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742212_1388 (size=31105) 2024-12-09T10:42:26,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-09T10:42:26,569 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/B/c35118f7f47b4a2f9d687bf80e6943df as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B/c35118f7f47b4a2f9d687bf80e6943df 2024-12-09T10:42:26,572 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ec3eeaafd032b9a023fe519c0fcbb0c5#A#compaction#325 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:42:26,573 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/A/6397868ca4464099a3b5d10f70f0a658 is 175, key is test_row_0/A:col10/1733740943614/Put/seqid=0 2024-12-09T10:42:26,574 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ec3eeaafd032b9a023fe519c0fcbb0c5/B of ec3eeaafd032b9a023fe519c0fcbb0c5 into c35118f7f47b4a2f9d687bf80e6943df(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:42:26,574 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ec3eeaafd032b9a023fe519c0fcbb0c5: 2024-12-09T10:42:26,574 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., storeName=ec3eeaafd032b9a023fe519c0fcbb0c5/B, priority=13, startTime=1733740946144; duration=0sec 2024-12-09T10:42:26,574 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T10:42:26,574 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ec3eeaafd032b9a023fe519c0fcbb0c5:B 2024-12-09T10:42:26,574 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T10:42:26,577 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T10:42:26,578 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1541): ec3eeaafd032b9a023fe519c0fcbb0c5/C is initiating minor compaction (all files) 2024-12-09T10:42:26,578 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ec3eeaafd032b9a023fe519c0fcbb0c5/C in TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. 2024-12-09T10:42:26,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742213_1389 (size=31549) 2024-12-09T10:42:26,578 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C/223a66ea80134e0f92de8ebaff18ef51, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C/1b126eba30ac4329a300afaa3fe08ce3, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C/703c5e801b6c45d89cd46f0fd9d89740] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp, totalSize=35.9 K 2024-12-09T10:42:26,582 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 223a66ea80134e0f92de8ebaff18ef51, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1733740942762 2024-12-09T10:42:26,582 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b126eba30ac4329a300afaa3fe08ce3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1733740942913 2024-12-09T10:42:26,582 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 703c5e801b6c45d89cd46f0fd9d89740, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1733740943614 2024-12-09T10:42:26,587 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ec3eeaafd032b9a023fe519c0fcbb0c5#C#compaction#327 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:42:26,588 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/C/f388b813d2e74481ac8eb84d9fc4c3c6 is 50, key is test_row_0/C:col10/1733740943614/Put/seqid=0 2024-12-09T10:42:26,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742214_1390 (size=12595) 2024-12-09T10:42:26,622 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=230, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/A/ee5e60d6a20c47e49656a630a45c45b2 2024-12-09T10:42:26,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/B/bcb1d2856e30474896617ee752d78eca is 50, key is test_row_0/B:col10/1733740944859/Put/seqid=0 2024-12-09T10:42:26,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742215_1391 (size=12151) 2024-12-09T10:42:26,982 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/A/6397868ca4464099a3b5d10f70f0a658 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/6397868ca4464099a3b5d10f70f0a658 2024-12-09T10:42:26,987 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ec3eeaafd032b9a023fe519c0fcbb0c5/A of ec3eeaafd032b9a023fe519c0fcbb0c5 into 6397868ca4464099a3b5d10f70f0a658(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:42:26,987 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ec3eeaafd032b9a023fe519c0fcbb0c5: 2024-12-09T10:42:26,987 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., storeName=ec3eeaafd032b9a023fe519c0fcbb0c5/A, priority=13, startTime=1733740946144; duration=0sec 2024-12-09T10:42:26,987 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:42:26,987 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ec3eeaafd032b9a023fe519c0fcbb0c5:A 2024-12-09T10:42:26,995 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/C/f388b813d2e74481ac8eb84d9fc4c3c6 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C/f388b813d2e74481ac8eb84d9fc4c3c6 2024-12-09T10:42:26,999 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ec3eeaafd032b9a023fe519c0fcbb0c5/C of ec3eeaafd032b9a023fe519c0fcbb0c5 into f388b813d2e74481ac8eb84d9fc4c3c6(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:42:26,999 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ec3eeaafd032b9a023fe519c0fcbb0c5: 2024-12-09T10:42:26,999 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., storeName=ec3eeaafd032b9a023fe519c0fcbb0c5/C, priority=13, startTime=1733740946144; duration=0sec 2024-12-09T10:42:26,999 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:42:26,999 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ec3eeaafd032b9a023fe519c0fcbb0c5:C 2024-12-09T10:42:27,033 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/B/bcb1d2856e30474896617ee752d78eca 2024-12-09T10:42:27,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/C/a8277585d07b4b248e52fe774de4aba0 is 50, key is test_row_0/C:col10/1733740944859/Put/seqid=0 2024-12-09T10:42:27,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742216_1392 (size=12151) 2024-12-09T10:42:27,045 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/C/a8277585d07b4b248e52fe774de4aba0 2024-12-09T10:42:27,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/A/ee5e60d6a20c47e49656a630a45c45b2 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/ee5e60d6a20c47e49656a630a45c45b2 2024-12-09T10:42:27,054 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/ee5e60d6a20c47e49656a630a45c45b2, entries=150, sequenceid=230, filesize=30.4 K 2024-12-09T10:42:27,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/B/bcb1d2856e30474896617ee752d78eca as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B/bcb1d2856e30474896617ee752d78eca 2024-12-09T10:42:27,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,059 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B/bcb1d2856e30474896617ee752d78eca, entries=150, sequenceid=230, filesize=11.9 K 2024-12-09T10:42:27,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/C/a8277585d07b4b248e52fe774de4aba0 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C/a8277585d07b4b248e52fe774de4aba0 2024-12-09T10:42:27,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,063 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C/a8277585d07b4b248e52fe774de4aba0, entries=150, sequenceid=230, filesize=11.9 K 2024-12-09T10:42:27,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,064 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=0 B/0 for ec3eeaafd032b9a023fe519c0fcbb0c5 in 872ms, sequenceid=230, compaction requested=false 2024-12-09T10:42:27,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for ec3eeaafd032b9a023fe519c0fcbb0c5: 2024-12-09T10:42:27,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. 2024-12-09T10:42:27,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-12-09T10:42:27,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-12-09T10:42:27,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,068 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-12-09T10:42:27,068 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7940 sec 2024-12-09T10:42:27,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,071 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 1.7980 sec 2024-12-09T10:42:27,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(8855): Flush requested on ec3eeaafd032b9a023fe519c0fcbb0c5 2024-12-09T10:42:27,170 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ec3eeaafd032b9a023fe519c0fcbb0c5 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-09T10:42:27,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec3eeaafd032b9a023fe519c0fcbb0c5, store=A 2024-12-09T10:42:27,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:42:27,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec3eeaafd032b9a023fe519c0fcbb0c5, store=B 2024-12-09T10:42:27,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:42:27,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec3eeaafd032b9a023fe519c0fcbb0c5, store=C 2024-12-09T10:42:27,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:42:27,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,194 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209e3d140fb687e4205a5c4d4e62ee047ad_ec3eeaafd032b9a023fe519c0fcbb0c5 is 50, key is test_row_0/A:col10/1733740947163/Put/seqid=0 2024-12-09T10:42:27,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742217_1393 (size=22268) 2024-12-09T10:42:27,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,201 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,205 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209e3d140fb687e4205a5c4d4e62ee047ad_ec3eeaafd032b9a023fe519c0fcbb0c5 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209e3d140fb687e4205a5c4d4e62ee047ad_ec3eeaafd032b9a023fe519c0fcbb0c5 2024-12-09T10:42:27,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,207 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/A/9370657e140c4b1b9ee49472a521415f, store: [table=TestAcidGuarantees family=A region=ec3eeaafd032b9a023fe519c0fcbb0c5] 2024-12-09T10:42:27,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,209 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/A/9370657e140c4b1b9ee49472a521415f is 175, key is test_row_0/A:col10/1733740947163/Put/seqid=0 2024-12-09T10:42:27,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742218_1394 (size=65673) 2024-12-09T10:42:27,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:27,257 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733740947231, max timestamp=1733740947254Num uniques 859; 2024-12-09T10:42:27,257 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733740947231, max timestamp=1733740947254Num uniques 859; 2024-12-09T10:42:27,257 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-09T10:42:27,257 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-09T10:42:27,258 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733740947231, max timestamp=1733740947254Num uniques 859; 2024-12-09T10:42:27,258 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-09T10:42:27,296 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:27,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50962 deadline: 1733741007280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:27,297 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:27,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50942 deadline: 1733741007285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:27,297 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:27,297 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:27,297 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:27,297 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:27,297 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:27,297 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:27,300 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:27,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50974 deadline: 1733741007286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:27,301 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:27,301 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:27,301 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:27,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50928 deadline: 1733741007289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:27,301 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:27,301 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:27,301 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:27,301 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:27,311 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:27,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50954 deadline: 1733741007296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:27,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:27,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:27,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:27,407 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:27,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50962 deadline: 1733741007399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:27,407 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:27,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50942 deadline: 1733741007399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:27,407 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:27,407 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:27,408 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:27,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:27,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:27,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:27,423 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:27,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50974 deadline: 1733741007410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:27,423 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:27,423 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:27,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50928 deadline: 1733741007410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:27,423 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:27,423 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:27,423 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:27,424 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:27,424 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:27,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-09T10:42:27,430 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T10:42:27,430 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-09T10:42:27,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-12-09T10:42:27,431 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:27,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50954 deadline: 1733741007419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:27,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-09T10:42:27,432 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:27,432 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:27,432 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T10:42:27,432 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:27,433 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T10:42:27,433 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T10:42:27,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-09T10:42:27,586 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-09T10:42:27,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. 2024-12-09T10:42:27,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. as already flushing 2024-12-09T10:42:27,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. 2024-12-09T10:42:27,587 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => ec3eeaafd032b9a023fe519c0fcbb0c5, NAME => 'TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:27,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => ec3eeaafd032b9a023fe519c0fcbb0c5, NAME => 'TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:27,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ec3eeaafd032b9a023fe519c0fcbb0c5, NAME => 'TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ec3eeaafd032b9a023fe519c0fcbb0c5, NAME => 'TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:27,621 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:27,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50962 deadline: 1733741007610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:27,622 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:27,622 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:27,622 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:27,629 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:27,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50942 deadline: 1733741007619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:27,630 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:27,630 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:27,630 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:27,635 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=243, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/A/9370657e140c4b1b9ee49472a521415f 2024-12-09T10:42:27,637 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:27,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50974 deadline: 1733741007629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:27,637 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:27,637 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:27,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50928 deadline: 1733741007629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:27,637 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:27,637 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:27,638 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:27,638 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:27,638 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:27,642 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/B/918816d6ffaa448ebcbb3ef3948a91c6 is 50, key is test_row_0/B:col10/1733740947163/Put/seqid=0 2024-12-09T10:42:27,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742219_1395 (size=12151) 2024-12-09T10:42:27,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:27,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50954 deadline: 1733741007640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:27,658 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:27,658 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:27,658 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:27,739 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-09T10:42:27,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. 2024-12-09T10:42:27,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. as already flushing 2024-12-09T10:42:27,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. 2024-12-09T10:42:27,740 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => ec3eeaafd032b9a023fe519c0fcbb0c5, NAME => 'TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:27,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => ec3eeaafd032b9a023fe519c0fcbb0c5, NAME => 'TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:27,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ec3eeaafd032b9a023fe519c0fcbb0c5, NAME => 'TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ec3eeaafd032b9a023fe519c0fcbb0c5, NAME => 'TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:27,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-09T10:42:27,893 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-09T10:42:27,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. 2024-12-09T10:42:27,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. as already flushing 2024-12-09T10:42:27,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. 2024-12-09T10:42:27,893 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => ec3eeaafd032b9a023fe519c0fcbb0c5, NAME => 'TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:27,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => ec3eeaafd032b9a023fe519c0fcbb0c5, NAME => 'TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:27,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ec3eeaafd032b9a023fe519c0fcbb0c5, NAME => 'TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ec3eeaafd032b9a023fe519c0fcbb0c5, NAME => 'TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:27,939 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:27,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50962 deadline: 1733741007929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:27,940 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:27,940 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:27,940 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:27,943 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:27,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50942 deadline: 1733741007940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:27,943 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:27,943 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:27,943 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:27,943 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:27,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50974 deadline: 1733741007940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:27,944 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:27,944 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:27,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50928 deadline: 1733741007940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:27,944 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:27,944 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:27,944 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:27,944 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:27,944 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:27,969 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:27,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50954 deadline: 1733741007959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:27,969 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:27,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:27,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:28,046 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-09T10:42:28,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. 2024-12-09T10:42:28,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. as already flushing 2024-12-09T10:42:28,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. 2024-12-09T10:42:28,047 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => ec3eeaafd032b9a023fe519c0fcbb0c5, NAME => 'TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:28,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => ec3eeaafd032b9a023fe519c0fcbb0c5, NAME => 'TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:28,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ec3eeaafd032b9a023fe519c0fcbb0c5, NAME => 'TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ec3eeaafd032b9a023fe519c0fcbb0c5, NAME => 'TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:28,058 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/B/918816d6ffaa448ebcbb3ef3948a91c6 2024-12-09T10:42:28,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-09T10:42:28,067 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/C/6015084bdc7b4d4fa8cea2ad49728a49 is 50, key is test_row_0/C:col10/1733740947163/Put/seqid=0 2024-12-09T10:42:28,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742220_1396 (size=12151) 2024-12-09T10:42:28,198 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-09T10:42:28,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. 2024-12-09T10:42:28,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. as already flushing 2024-12-09T10:42:28,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. 2024-12-09T10:42:28,199 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => ec3eeaafd032b9a023fe519c0fcbb0c5, NAME => 'TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:28,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => ec3eeaafd032b9a023fe519c0fcbb0c5, NAME => 'TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:28,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ec3eeaafd032b9a023fe519c0fcbb0c5, NAME => 'TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ec3eeaafd032b9a023fe519c0fcbb0c5, NAME => 'TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:28,352 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-09T10:42:28,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. 2024-12-09T10:42:28,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. as already flushing 2024-12-09T10:42:28,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. 2024-12-09T10:42:28,352 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => ec3eeaafd032b9a023fe519c0fcbb0c5, NAME => 'TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:28,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => ec3eeaafd032b9a023fe519c0fcbb0c5, NAME => 'TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:28,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ec3eeaafd032b9a023fe519c0fcbb0c5, NAME => 'TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ec3eeaafd032b9a023fe519c0fcbb0c5, NAME => 'TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:28,465 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:28,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50962 deadline: 1733741008449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:28,466 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:28,466 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:28,466 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:28,466 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:28,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50942 deadline: 1733741008450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:28,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:28,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:28,466 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:28,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:28,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50974 deadline: 1733741008450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:28,467 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:28,467 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:28,467 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:28,467 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:28,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50928 deadline: 1733741008450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:28,467 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:28,467 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:28,467 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:28,472 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/C/6015084bdc7b4d4fa8cea2ad49728a49 2024-12-09T10:42:28,476 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/A/9370657e140c4b1b9ee49472a521415f as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/9370657e140c4b1b9ee49472a521415f 2024-12-09T10:42:28,479 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/9370657e140c4b1b9ee49472a521415f, entries=350, sequenceid=243, filesize=64.1 K 2024-12-09T10:42:28,480 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/B/918816d6ffaa448ebcbb3ef3948a91c6 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B/918816d6ffaa448ebcbb3ef3948a91c6 2024-12-09T10:42:28,483 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B/918816d6ffaa448ebcbb3ef3948a91c6, entries=150, sequenceid=243, filesize=11.9 K 2024-12-09T10:42:28,484 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/C/6015084bdc7b4d4fa8cea2ad49728a49 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C/6015084bdc7b4d4fa8cea2ad49728a49 2024-12-09T10:42:28,487 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C/6015084bdc7b4d4fa8cea2ad49728a49, entries=150, sequenceid=243, filesize=11.9 K 2024-12-09T10:42:28,488 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for ec3eeaafd032b9a023fe519c0fcbb0c5 in 1318ms, sequenceid=243, compaction requested=true 2024-12-09T10:42:28,488 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ec3eeaafd032b9a023fe519c0fcbb0c5: 2024-12-09T10:42:28,488 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T10:42:28,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ec3eeaafd032b9a023fe519c0fcbb0c5:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T10:42:28,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:42:28,489 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T10:42:28,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ec3eeaafd032b9a023fe519c0fcbb0c5:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T10:42:28,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:42:28,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ec3eeaafd032b9a023fe519c0fcbb0c5:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T10:42:28,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T10:42:28,491 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 128327 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T10:42:28,491 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1541): ec3eeaafd032b9a023fe519c0fcbb0c5/A is initiating minor compaction (all files) 2024-12-09T10:42:28,491 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ec3eeaafd032b9a023fe519c0fcbb0c5/A in TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. 2024-12-09T10:42:28,491 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/6397868ca4464099a3b5d10f70f0a658, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/ee5e60d6a20c47e49656a630a45c45b2, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/9370657e140c4b1b9ee49472a521415f] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp, totalSize=125.3 K 2024-12-09T10:42:28,491 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T10:42:28,491 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. 2024-12-09T10:42:28,491 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1541): ec3eeaafd032b9a023fe519c0fcbb0c5/B is initiating minor compaction (all files) 2024-12-09T10:42:28,491 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. files: [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/6397868ca4464099a3b5d10f70f0a658, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/ee5e60d6a20c47e49656a630a45c45b2, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/9370657e140c4b1b9ee49472a521415f] 2024-12-09T10:42:28,491 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ec3eeaafd032b9a023fe519c0fcbb0c5/B in TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. 2024-12-09T10:42:28,491 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B/c35118f7f47b4a2f9d687bf80e6943df, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B/bcb1d2856e30474896617ee752d78eca, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B/918816d6ffaa448ebcbb3ef3948a91c6] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp, totalSize=36.0 K 2024-12-09T10:42:28,491 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6397868ca4464099a3b5d10f70f0a658, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1733740943614 2024-12-09T10:42:28,492 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting c35118f7f47b4a2f9d687bf80e6943df, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1733740943614 2024-12-09T10:42:28,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(8855): Flush requested on ec3eeaafd032b9a023fe519c0fcbb0c5 2024-12-09T10:42:28,492 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting ee5e60d6a20c47e49656a630a45c45b2, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1733740944859 2024-12-09T10:42:28,492 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting bcb1d2856e30474896617ee752d78eca, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1733740944859 2024-12-09T10:42:28,492 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ec3eeaafd032b9a023fe519c0fcbb0c5 3/3 column families, dataSize=147.60 KB heapSize=410.77 KB 2024-12-09T10:42:28,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec3eeaafd032b9a023fe519c0fcbb0c5, store=A 2024-12-09T10:42:28,492 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9370657e140c4b1b9ee49472a521415f, keycount=350, bloomtype=ROW, size=64.1 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1733740947150 2024-12-09T10:42:28,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T10:42:28,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec3eeaafd032b9a023fe519c0fcbb0c5, store=B 2024-12-09T10:42:28,492 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 918816d6ffaa448ebcbb3ef3948a91c6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1733740947150 2024-12-09T10:42:28,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T10:42:28,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec3eeaafd032b9a023fe519c0fcbb0c5, store=C 2024-12-09T10:42:28,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T10:42:28,500 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ec3eeaafd032b9a023fe519c0fcbb0c5#B#compaction#334 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:42:28,501 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/B/1d382ca18620461d8083d110ba832dc2 is 50, key is test_row_0/B:col10/1733740947163/Put/seqid=0 2024-12-09T10:42:28,501 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120959a48fa660544664b44e8f222406b1dd_ec3eeaafd032b9a023fe519c0fcbb0c5 is 50, key is test_row_0/A:col10/1733740948490/Put/seqid=0 2024-12-09T10:42:28,502 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ec3eeaafd032b9a023fe519c0fcbb0c5] 2024-12-09T10:42:28,504 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-09T10:42:28,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. 2024-12-09T10:42:28,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. as already flushing 2024-12-09T10:42:28,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. 2024-12-09T10:42:28,505 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => ec3eeaafd032b9a023fe519c0fcbb0c5, NAME => 'TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:28,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => ec3eeaafd032b9a023fe519c0fcbb0c5, NAME => 'TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:28,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ec3eeaafd032b9a023fe519c0fcbb0c5, NAME => 'TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ec3eeaafd032b9a023fe519c0fcbb0c5, NAME => 'TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:28,511 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241209588cb51fd877492d90ec90b788bd78c1_ec3eeaafd032b9a023fe519c0fcbb0c5 store=[table=TestAcidGuarantees family=A region=ec3eeaafd032b9a023fe519c0fcbb0c5] 2024-12-09T10:42:28,513 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241209588cb51fd877492d90ec90b788bd78c1_ec3eeaafd032b9a023fe519c0fcbb0c5, store=[table=TestAcidGuarantees family=A region=ec3eeaafd032b9a023fe519c0fcbb0c5] 2024-12-09T10:42:28,513 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209588cb51fd877492d90ec90b788bd78c1_ec3eeaafd032b9a023fe519c0fcbb0c5 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ec3eeaafd032b9a023fe519c0fcbb0c5] 2024-12-09T10:42:28,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742222_1398 (size=14994) 2024-12-09T10:42:28,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742221_1397 (size=12697) 2024-12-09T10:42:28,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742223_1399 (size=4469) 2024-12-09T10:42:28,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-09T10:42:28,580 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:28,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50954 deadline: 1733741008567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:28,580 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:28,580 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:28,580 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:28,658 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-09T10:42:28,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. 2024-12-09T10:42:28,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. as already flushing 2024-12-09T10:42:28,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. 2024-12-09T10:42:28,658 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => ec3eeaafd032b9a023fe519c0fcbb0c5, NAME => 'TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:28,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => ec3eeaafd032b9a023fe519c0fcbb0c5, NAME => 'TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:28,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ec3eeaafd032b9a023fe519c0fcbb0c5, NAME => 'TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ec3eeaafd032b9a023fe519c0fcbb0c5, NAME => 'TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:28,697 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:28,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50954 deadline: 1733741008690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:28,698 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:28,698 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:28,698 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:28,811 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-09T10:42:28,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. 2024-12-09T10:42:28,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. as already flushing 2024-12-09T10:42:28,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. 2024-12-09T10:42:28,811 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => ec3eeaafd032b9a023fe519c0fcbb0c5, NAME => 'TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:28,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => ec3eeaafd032b9a023fe519c0fcbb0c5, NAME => 'TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:28,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ec3eeaafd032b9a023fe519c0fcbb0c5, NAME => 'TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ec3eeaafd032b9a023fe519c0fcbb0c5, NAME => 'TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:28,912 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:28,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50954 deadline: 1733741008910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:28,913 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:28,913 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:28,913 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:28,924 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:28,928 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120959a48fa660544664b44e8f222406b1dd_ec3eeaafd032b9a023fe519c0fcbb0c5 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120959a48fa660544664b44e8f222406b1dd_ec3eeaafd032b9a023fe519c0fcbb0c5 2024-12-09T10:42:28,930 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/A/5a8533ef2d7242239c9b0e4d41375dad, store: [table=TestAcidGuarantees family=A region=ec3eeaafd032b9a023fe519c0fcbb0c5] 2024-12-09T10:42:28,930 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/B/1d382ca18620461d8083d110ba832dc2 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B/1d382ca18620461d8083d110ba832dc2 2024-12-09T10:42:28,930 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ec3eeaafd032b9a023fe519c0fcbb0c5#A#compaction#335 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:42:28,930 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/A/5a8533ef2d7242239c9b0e4d41375dad is 175, key is test_row_0/A:col10/1733740948490/Put/seqid=0 2024-12-09T10:42:28,931 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/A/078a6e04b2134260a3e6517d8836972e is 175, key is test_row_0/A:col10/1733740947163/Put/seqid=0 2024-12-09T10:42:28,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742224_1400 (size=39949) 2024-12-09T10:42:28,935 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=279, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/A/5a8533ef2d7242239c9b0e4d41375dad 2024-12-09T10:42:28,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742225_1401 (size=31651) 2024-12-09T10:42:28,938 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ec3eeaafd032b9a023fe519c0fcbb0c5/B of ec3eeaafd032b9a023fe519c0fcbb0c5 into 1d382ca18620461d8083d110ba832dc2(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:42:28,938 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ec3eeaafd032b9a023fe519c0fcbb0c5: 2024-12-09T10:42:28,938 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., storeName=ec3eeaafd032b9a023fe519c0fcbb0c5/B, priority=13, startTime=1733740948489; duration=0sec 2024-12-09T10:42:28,938 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T10:42:28,938 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ec3eeaafd032b9a023fe519c0fcbb0c5:B 2024-12-09T10:42:28,938 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T10:42:28,939 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T10:42:28,940 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1541): ec3eeaafd032b9a023fe519c0fcbb0c5/C is initiating minor compaction (all files) 2024-12-09T10:42:28,940 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ec3eeaafd032b9a023fe519c0fcbb0c5/C in TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. 2024-12-09T10:42:28,940 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C/f388b813d2e74481ac8eb84d9fc4c3c6, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C/a8277585d07b4b248e52fe774de4aba0, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C/6015084bdc7b4d4fa8cea2ad49728a49] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp, totalSize=36.0 K 2024-12-09T10:42:28,940 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting f388b813d2e74481ac8eb84d9fc4c3c6, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1733740943614 2024-12-09T10:42:28,940 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting a8277585d07b4b248e52fe774de4aba0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1733740944859 2024-12-09T10:42:28,941 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 6015084bdc7b4d4fa8cea2ad49728a49, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1733740947150 2024-12-09T10:42:28,943 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/B/0593c55c633a40cfa184846b75cd3395 is 50, key is test_row_0/B:col10/1733740948490/Put/seqid=0 2024-12-09T10:42:28,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742226_1402 (size=12301) 2024-12-09T10:42:28,951 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ec3eeaafd032b9a023fe519c0fcbb0c5#C#compaction#337 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:42:28,951 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/B/0593c55c633a40cfa184846b75cd3395 2024-12-09T10:42:28,952 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/C/6dd193f1440e4cd5b76b259169c4c5ef is 50, key is test_row_0/C:col10/1733740947163/Put/seqid=0 2024-12-09T10:42:28,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742227_1403 (size=12697) 2024-12-09T10:42:28,959 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/C/1f83b2c121824ce5b2eef10ca764bb80 is 50, key is test_row_0/C:col10/1733740948490/Put/seqid=0 2024-12-09T10:42:28,964 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-09T10:42:28,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. 2024-12-09T10:42:28,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. as already flushing 2024-12-09T10:42:28,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. 2024-12-09T10:42:28,965 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => ec3eeaafd032b9a023fe519c0fcbb0c5, NAME => 'TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:28,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => ec3eeaafd032b9a023fe519c0fcbb0c5, NAME => 'TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:28,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ec3eeaafd032b9a023fe519c0fcbb0c5, NAME => 'TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ec3eeaafd032b9a023fe519c0fcbb0c5, NAME => 'TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:28,968 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/C/6dd193f1440e4cd5b76b259169c4c5ef as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C/6dd193f1440e4cd5b76b259169c4c5ef 2024-12-09T10:42:28,972 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ec3eeaafd032b9a023fe519c0fcbb0c5/C of ec3eeaafd032b9a023fe519c0fcbb0c5 into 6dd193f1440e4cd5b76b259169c4c5ef(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:42:28,972 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ec3eeaafd032b9a023fe519c0fcbb0c5: 2024-12-09T10:42:28,972 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., storeName=ec3eeaafd032b9a023fe519c0fcbb0c5/C, priority=13, startTime=1733740948489; duration=0sec 2024-12-09T10:42:28,972 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:42:28,972 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ec3eeaafd032b9a023fe519c0fcbb0c5:C 2024-12-09T10:42:28,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742228_1404 (size=12301) 2024-12-09T10:42:29,117 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-09T10:42:29,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. 2024-12-09T10:42:29,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. as already flushing 2024-12-09T10:42:29,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. 2024-12-09T10:42:29,118 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => ec3eeaafd032b9a023fe519c0fcbb0c5, NAME => 'TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:29,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => ec3eeaafd032b9a023fe519c0fcbb0c5, NAME => 'TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:29,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ec3eeaafd032b9a023fe519c0fcbb0c5, NAME => 'TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ec3eeaafd032b9a023fe519c0fcbb0c5, NAME => 'TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:29,229 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:29,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50954 deadline: 1733741009220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:29,230 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:29,230 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:29,230 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:29,271 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-09T10:42:29,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. 2024-12-09T10:42:29,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. as already flushing 2024-12-09T10:42:29,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. 2024-12-09T10:42:29,271 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => ec3eeaafd032b9a023fe519c0fcbb0c5, NAME => 'TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:29,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => ec3eeaafd032b9a023fe519c0fcbb0c5, NAME => 'TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:29,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ec3eeaafd032b9a023fe519c0fcbb0c5, NAME => 'TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ec3eeaafd032b9a023fe519c0fcbb0c5, NAME => 'TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:29,341 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/A/078a6e04b2134260a3e6517d8836972e as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/078a6e04b2134260a3e6517d8836972e 2024-12-09T10:42:29,345 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ec3eeaafd032b9a023fe519c0fcbb0c5/A of ec3eeaafd032b9a023fe519c0fcbb0c5 into 078a6e04b2134260a3e6517d8836972e(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:42:29,345 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ec3eeaafd032b9a023fe519c0fcbb0c5: 2024-12-09T10:42:29,345 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., storeName=ec3eeaafd032b9a023fe519c0fcbb0c5/A, priority=13, startTime=1733740948488; duration=0sec 2024-12-09T10:42:29,345 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:42:29,345 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ec3eeaafd032b9a023fe519c0fcbb0c5:A 2024-12-09T10:42:29,375 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/C/1f83b2c121824ce5b2eef10ca764bb80 2024-12-09T10:42:29,380 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/A/5a8533ef2d7242239c9b0e4d41375dad as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/5a8533ef2d7242239c9b0e4d41375dad 2024-12-09T10:42:29,383 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/5a8533ef2d7242239c9b0e4d41375dad, entries=200, sequenceid=279, filesize=39.0 K 2024-12-09T10:42:29,384 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/B/0593c55c633a40cfa184846b75cd3395 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B/0593c55c633a40cfa184846b75cd3395 2024-12-09T10:42:29,387 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B/0593c55c633a40cfa184846b75cd3395, entries=150, sequenceid=279, filesize=12.0 K 2024-12-09T10:42:29,388 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/C/1f83b2c121824ce5b2eef10ca764bb80 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C/1f83b2c121824ce5b2eef10ca764bb80 2024-12-09T10:42:29,391 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C/1f83b2c121824ce5b2eef10ca764bb80, entries=150, sequenceid=279, filesize=12.0 K 2024-12-09T10:42:29,392 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~410.72 KB/420576, currentSize=40.25 KB/41220 for ec3eeaafd032b9a023fe519c0fcbb0c5 in 900ms, sequenceid=279, compaction requested=false 2024-12-09T10:42:29,392 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ec3eeaafd032b9a023fe519c0fcbb0c5: 2024-12-09T10:42:29,423 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-09T10:42:29,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. 2024-12-09T10:42:29,424 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing ec3eeaafd032b9a023fe519c0fcbb0c5 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-12-09T10:42:29,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec3eeaafd032b9a023fe519c0fcbb0c5, store=A 2024-12-09T10:42:29,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:42:29,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec3eeaafd032b9a023fe519c0fcbb0c5, store=B 2024-12-09T10:42:29,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:42:29,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec3eeaafd032b9a023fe519c0fcbb0c5, store=C 2024-12-09T10:42:29,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:42:29,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412096048e11995ec4bdf871b146a9a41a78d_ec3eeaafd032b9a023fe519c0fcbb0c5 is 50, key is test_row_0/A:col10/1733740948566/Put/seqid=0 2024-12-09T10:42:29,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742229_1405 (size=12454) 2024-12-09T10:42:29,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(8855): Flush requested on ec3eeaafd032b9a023fe519c0fcbb0c5 2024-12-09T10:42:29,481 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. as already flushing 2024-12-09T10:42:29,523 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733740949494, max timestamp=1733740949520Num uniques 859; 2024-12-09T10:42:29,523 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T10:42:29,523 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733740949494, max timestamp=1733740949520Num uniques 859; 2024-12-09T10:42:29,523 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733740949494, max timestamp=1733740949520Num uniques 859; 2024-12-09T10:42:29,523 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T10:42:29,523 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T10:42:29,574 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:29,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50942 deadline: 1733741009572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:29,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:29,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:29,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:29,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-09T10:42:29,587 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:29,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50928 deadline: 1733741009573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:29,588 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:29,588 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:29,588 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:29,588 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:29,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50962 deadline: 1733741009573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:29,589 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:29,589 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:29,589 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:29,593 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:29,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50974 deadline: 1733741009575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:29,593 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:29,594 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:29,594 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:29,691 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:29,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50942 deadline: 1733741009679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:29,692 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:29,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:29,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:29,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50928 deadline: 1733741009689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:29,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:29,692 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:29,692 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:29,692 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:29,707 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:29,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50962 deadline: 1733741009700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:29,707 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:29,707 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:29,707 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:29,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50974 deadline: 1733741009700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:29,708 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:29,708 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:29,708 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:29,708 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:29,751 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:29,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50954 deadline: 1733741009739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:29,752 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:29,752 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:29,752 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:29,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:29,840 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412096048e11995ec4bdf871b146a9a41a78d_ec3eeaafd032b9a023fe519c0fcbb0c5 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412096048e11995ec4bdf871b146a9a41a78d_ec3eeaafd032b9a023fe519c0fcbb0c5 2024-12-09T10:42:29,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/A/9e5164d6cc1d42eea48550bd4611704f, store: [table=TestAcidGuarantees family=A region=ec3eeaafd032b9a023fe519c0fcbb0c5] 2024-12-09T10:42:29,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/A/9e5164d6cc1d42eea48550bd4611704f is 175, key is test_row_0/A:col10/1733740948566/Put/seqid=0 2024-12-09T10:42:29,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742230_1406 (size=31255) 2024-12-09T10:42:29,851 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=291, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/A/9e5164d6cc1d42eea48550bd4611704f 2024-12-09T10:42:29,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/B/b2f43342d26d4010b207d658ac001cd0 is 50, key is test_row_0/B:col10/1733740948566/Put/seqid=0 2024-12-09T10:42:29,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742231_1407 (size=12301) 2024-12-09T10:42:29,908 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:29,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50942 deadline: 1733741009900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:29,909 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:29,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50928 deadline: 1733741009900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:29,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:29,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:29,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:29,909 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:29,909 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:29,909 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:29,924 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:29,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50962 deadline: 1733741009919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:29,925 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:29,925 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:29,925 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:29,933 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:29,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50974 deadline: 1733741009919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:29,934 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:29,934 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:29,934 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:30,224 WARN [regionserver/7ea43b39fc24:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 6, running: 1 2024-12-09T10:42:30,224 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7ea43b39fc24:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/109a22e8b6744c7d82d4e0268b1f411c, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/6b2d64585f4c41dabb2b98b13a2c093a, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/865db025f7b64c9f8cc118aea566ff6e, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/ee5ffe6bb7b1459ca5355cd0dcdeb66b, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/79aa314b74184b4e8cd8009eb011b43b, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/b385f56d32984e1e930315290ecdc5d1, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/7fe5f09840f84199a2b78fe54d3c2347, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/fff415a458144776ab8ba5d7c58d9ed9, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/d842be8dc62b4bad889d09b5176025c4, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/8f147ed875d842ed955e986f85ff670b, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/51d4cbed64d44502b1bf112f90605b46, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/26d5c24373ad4374b4a561f52633c68e, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/3344d67626d84d8193d8b75dafc6b296, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/6397868ca4464099a3b5d10f70f0a658, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/ee5e60d6a20c47e49656a630a45c45b2, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/9370657e140c4b1b9ee49472a521415f] to archive 2024-12-09T10:42:30,225 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7ea43b39fc24:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T10:42:30,226 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7ea43b39fc24:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/109a22e8b6744c7d82d4e0268b1f411c to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/109a22e8b6744c7d82d4e0268b1f411c 2024-12-09T10:42:30,228 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7ea43b39fc24:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/6b2d64585f4c41dabb2b98b13a2c093a to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/6b2d64585f4c41dabb2b98b13a2c093a 2024-12-09T10:42:30,229 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7ea43b39fc24:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/865db025f7b64c9f8cc118aea566ff6e to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/865db025f7b64c9f8cc118aea566ff6e 2024-12-09T10:42:30,230 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:30,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50942 deadline: 1733741010220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:30,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:30,230 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:30,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:30,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:30,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50928 deadline: 1733741010220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:30,231 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:30,231 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:30,231 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:30,231 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7ea43b39fc24:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/ee5ffe6bb7b1459ca5355cd0dcdeb66b to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/ee5ffe6bb7b1459ca5355cd0dcdeb66b 2024-12-09T10:42:30,232 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7ea43b39fc24:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/79aa314b74184b4e8cd8009eb011b43b to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/79aa314b74184b4e8cd8009eb011b43b 2024-12-09T10:42:30,233 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7ea43b39fc24:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/b385f56d32984e1e930315290ecdc5d1 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/b385f56d32984e1e930315290ecdc5d1 2024-12-09T10:42:30,234 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7ea43b39fc24:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/7fe5f09840f84199a2b78fe54d3c2347 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/7fe5f09840f84199a2b78fe54d3c2347 2024-12-09T10:42:30,234 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7ea43b39fc24:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/fff415a458144776ab8ba5d7c58d9ed9 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/fff415a458144776ab8ba5d7c58d9ed9 2024-12-09T10:42:30,235 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7ea43b39fc24:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/d842be8dc62b4bad889d09b5176025c4 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/d842be8dc62b4bad889d09b5176025c4 2024-12-09T10:42:30,236 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7ea43b39fc24:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/8f147ed875d842ed955e986f85ff670b to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/8f147ed875d842ed955e986f85ff670b 2024-12-09T10:42:30,237 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:30,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50962 deadline: 1733741010229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:30,237 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7ea43b39fc24:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/51d4cbed64d44502b1bf112f90605b46 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/51d4cbed64d44502b1bf112f90605b46 2024-12-09T10:42:30,237 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:30,237 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:30,237 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:30,238 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7ea43b39fc24:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/26d5c24373ad4374b4a561f52633c68e to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/26d5c24373ad4374b4a561f52633c68e 2024-12-09T10:42:30,239 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7ea43b39fc24:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/3344d67626d84d8193d8b75dafc6b296 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/3344d67626d84d8193d8b75dafc6b296 2024-12-09T10:42:30,240 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7ea43b39fc24:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/6397868ca4464099a3b5d10f70f0a658 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/6397868ca4464099a3b5d10f70f0a658 2024-12-09T10:42:30,241 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7ea43b39fc24:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/ee5e60d6a20c47e49656a630a45c45b2 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/ee5e60d6a20c47e49656a630a45c45b2 2024-12-09T10:42:30,242 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7ea43b39fc24:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/9370657e140c4b1b9ee49472a521415f to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/9370657e140c4b1b9ee49472a521415f 2024-12-09T10:42:30,246 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7ea43b39fc24:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B/5c0aa3f183f244bc9bf673c22c016d3a, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B/a9462ad893a142239d9222053b5908cd, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B/6b0fad94b2e94bb38296072812e15691, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B/e05a4d8e7b73445f97adea5aa4b23f9e, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B/fb883fa67fae4f72a07d754226b50f0f, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B/0bb5884d15904307ba4074f165fa274e, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B/05dc4a75a0444d44ad1ff5a8b5d3a642, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B/e76673facc9e4d2a84856331c080b386, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B/5dac524d062d467c85b759e6f54d7e89, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B/01bee7bdac78474fbeace6788f12ddb3, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B/9b6bec719f7c483a86b583da919ea1b9, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B/5b1869c5cf294980bba672cd66b3b5a9, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B/c35118f7f47b4a2f9d687bf80e6943df, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B/a50f00f7126144e285937ccb895062fd, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B/bcb1d2856e30474896617ee752d78eca, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B/918816d6ffaa448ebcbb3ef3948a91c6] to archive 2024-12-09T10:42:30,246 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7ea43b39fc24:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T10:42:30,248 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7ea43b39fc24:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B/5c0aa3f183f244bc9bf673c22c016d3a to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B/5c0aa3f183f244bc9bf673c22c016d3a 2024-12-09T10:42:30,248 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7ea43b39fc24:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B/a9462ad893a142239d9222053b5908cd to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B/a9462ad893a142239d9222053b5908cd 2024-12-09T10:42:30,249 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7ea43b39fc24:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B/6b0fad94b2e94bb38296072812e15691 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B/6b0fad94b2e94bb38296072812e15691 2024-12-09T10:42:30,250 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7ea43b39fc24:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B/e05a4d8e7b73445f97adea5aa4b23f9e to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B/e05a4d8e7b73445f97adea5aa4b23f9e 2024-12-09T10:42:30,251 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7ea43b39fc24:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B/fb883fa67fae4f72a07d754226b50f0f to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B/fb883fa67fae4f72a07d754226b50f0f 2024-12-09T10:42:30,252 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7ea43b39fc24:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B/0bb5884d15904307ba4074f165fa274e to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B/0bb5884d15904307ba4074f165fa274e 2024-12-09T10:42:30,253 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7ea43b39fc24:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B/05dc4a75a0444d44ad1ff5a8b5d3a642 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B/05dc4a75a0444d44ad1ff5a8b5d3a642 2024-12-09T10:42:30,254 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:30,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50974 deadline: 1733741010239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:30,254 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7ea43b39fc24:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B/e76673facc9e4d2a84856331c080b386 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B/e76673facc9e4d2a84856331c080b386 2024-12-09T10:42:30,254 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:30,254 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:30,254 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:30,255 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7ea43b39fc24:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B/5dac524d062d467c85b759e6f54d7e89 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B/5dac524d062d467c85b759e6f54d7e89 2024-12-09T10:42:30,256 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7ea43b39fc24:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B/01bee7bdac78474fbeace6788f12ddb3 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B/01bee7bdac78474fbeace6788f12ddb3 2024-12-09T10:42:30,257 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7ea43b39fc24:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B/9b6bec719f7c483a86b583da919ea1b9 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B/9b6bec719f7c483a86b583da919ea1b9 2024-12-09T10:42:30,258 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7ea43b39fc24:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B/5b1869c5cf294980bba672cd66b3b5a9 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B/5b1869c5cf294980bba672cd66b3b5a9 2024-12-09T10:42:30,260 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7ea43b39fc24:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B/c35118f7f47b4a2f9d687bf80e6943df to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B/c35118f7f47b4a2f9d687bf80e6943df 2024-12-09T10:42:30,261 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/B/b2f43342d26d4010b207d658ac001cd0 2024-12-09T10:42:30,261 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7ea43b39fc24:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B/a50f00f7126144e285937ccb895062fd to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B/a50f00f7126144e285937ccb895062fd 2024-12-09T10:42:30,264 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7ea43b39fc24:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B/bcb1d2856e30474896617ee752d78eca to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B/bcb1d2856e30474896617ee752d78eca 2024-12-09T10:42:30,265 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7ea43b39fc24:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B/918816d6ffaa448ebcbb3ef3948a91c6 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B/918816d6ffaa448ebcbb3ef3948a91c6 2024-12-09T10:42:30,267 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7ea43b39fc24:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C/20d65f890c4f46c19b993aabc4feda36, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C/162225dfd7c443eca45d98b39020b374, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C/a844161280444e78b808ed6353abf0e6, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C/8ec70b07680a4f5bad18ed997afd9d10, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C/431e8973d54d4052911a079d70c72e48, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C/ac17b63080d040a29c2825c823ce4af6, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C/cb5d0776a4ee47678246794670559d94, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C/f3bee8c13f544f2693a7e46e8bc888fd, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C/b6f9ee9ca9a64f2b9e6c83b4414bffcc, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C/223a66ea80134e0f92de8ebaff18ef51, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C/a4ee336c554c4e849c0aed23c7e0ce63, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C/1b126eba30ac4329a300afaa3fe08ce3, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C/f388b813d2e74481ac8eb84d9fc4c3c6, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C/703c5e801b6c45d89cd46f0fd9d89740, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C/a8277585d07b4b248e52fe774de4aba0, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C/6015084bdc7b4d4fa8cea2ad49728a49] to archive 2024-12-09T10:42:30,268 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7ea43b39fc24:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T10:42:30,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/C/8831859619f144eb8d0264f580a14475 is 50, key is test_row_0/C:col10/1733740948566/Put/seqid=0 2024-12-09T10:42:30,269 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7ea43b39fc24:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C/20d65f890c4f46c19b993aabc4feda36 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C/20d65f890c4f46c19b993aabc4feda36 2024-12-09T10:42:30,270 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7ea43b39fc24:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C/162225dfd7c443eca45d98b39020b374 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C/162225dfd7c443eca45d98b39020b374 2024-12-09T10:42:30,272 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7ea43b39fc24:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C/a844161280444e78b808ed6353abf0e6 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C/a844161280444e78b808ed6353abf0e6 2024-12-09T10:42:30,273 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7ea43b39fc24:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C/8ec70b07680a4f5bad18ed997afd9d10 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C/8ec70b07680a4f5bad18ed997afd9d10 2024-12-09T10:42:30,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742232_1408 (size=12301) 2024-12-09T10:42:30,275 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7ea43b39fc24:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C/431e8973d54d4052911a079d70c72e48 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C/431e8973d54d4052911a079d70c72e48 2024-12-09T10:42:30,275 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7ea43b39fc24:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C/ac17b63080d040a29c2825c823ce4af6 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C/ac17b63080d040a29c2825c823ce4af6 2024-12-09T10:42:30,276 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7ea43b39fc24:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C/cb5d0776a4ee47678246794670559d94 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C/cb5d0776a4ee47678246794670559d94 2024-12-09T10:42:30,278 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7ea43b39fc24:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C/f3bee8c13f544f2693a7e46e8bc888fd to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C/f3bee8c13f544f2693a7e46e8bc888fd 2024-12-09T10:42:30,280 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7ea43b39fc24:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C/b6f9ee9ca9a64f2b9e6c83b4414bffcc to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C/b6f9ee9ca9a64f2b9e6c83b4414bffcc 2024-12-09T10:42:30,281 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7ea43b39fc24:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C/223a66ea80134e0f92de8ebaff18ef51 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C/223a66ea80134e0f92de8ebaff18ef51 2024-12-09T10:42:30,282 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7ea43b39fc24:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C/a4ee336c554c4e849c0aed23c7e0ce63 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C/a4ee336c554c4e849c0aed23c7e0ce63 2024-12-09T10:42:30,283 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7ea43b39fc24:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C/1b126eba30ac4329a300afaa3fe08ce3 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C/1b126eba30ac4329a300afaa3fe08ce3 2024-12-09T10:42:30,284 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7ea43b39fc24:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C/f388b813d2e74481ac8eb84d9fc4c3c6 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C/f388b813d2e74481ac8eb84d9fc4c3c6 2024-12-09T10:42:30,285 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7ea43b39fc24:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C/703c5e801b6c45d89cd46f0fd9d89740 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C/703c5e801b6c45d89cd46f0fd9d89740 2024-12-09T10:42:30,286 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7ea43b39fc24:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C/a8277585d07b4b248e52fe774de4aba0 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C/a8277585d07b4b248e52fe774de4aba0 2024-12-09T10:42:30,286 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7ea43b39fc24:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C/6015084bdc7b4d4fa8cea2ad49728a49 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C/6015084bdc7b4d4fa8cea2ad49728a49 2024-12-09T10:42:30,675 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/C/8831859619f144eb8d0264f580a14475 2024-12-09T10:42:30,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/A/9e5164d6cc1d42eea48550bd4611704f as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/9e5164d6cc1d42eea48550bd4611704f 2024-12-09T10:42:30,683 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/9e5164d6cc1d42eea48550bd4611704f, entries=150, sequenceid=291, filesize=30.5 K 2024-12-09T10:42:30,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/B/b2f43342d26d4010b207d658ac001cd0 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B/b2f43342d26d4010b207d658ac001cd0 2024-12-09T10:42:30,687 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B/b2f43342d26d4010b207d658ac001cd0, entries=150, sequenceid=291, filesize=12.0 K 2024-12-09T10:42:30,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/C/8831859619f144eb8d0264f580a14475 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C/8831859619f144eb8d0264f580a14475 2024-12-09T10:42:30,692 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C/8831859619f144eb8d0264f580a14475, entries=150, sequenceid=291, filesize=12.0 K 2024-12-09T10:42:30,693 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for ec3eeaafd032b9a023fe519c0fcbb0c5 in 1269ms, sequenceid=291, compaction requested=true 2024-12-09T10:42:30,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for ec3eeaafd032b9a023fe519c0fcbb0c5: 2024-12-09T10:42:30,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. 2024-12-09T10:42:30,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-12-09T10:42:30,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-12-09T10:42:30,696 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-12-09T10:42:30,696 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.2610 sec 2024-12-09T10:42:30,697 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 3.2660 sec 2024-12-09T10:42:30,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(8855): Flush requested on ec3eeaafd032b9a023fe519c0fcbb0c5 2024-12-09T10:42:30,748 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ec3eeaafd032b9a023fe519c0fcbb0c5 3/3 column families, dataSize=147.60 KB heapSize=420.84 KB 2024-12-09T10:42:30,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec3eeaafd032b9a023fe519c0fcbb0c5, store=A 2024-12-09T10:42:30,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-09T10:42:30,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec3eeaafd032b9a023fe519c0fcbb0c5, store=B 2024-12-09T10:42:30,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-09T10:42:30,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec3eeaafd032b9a023fe519c0fcbb0c5, store=C 2024-12-09T10:42:30,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-09T10:42:30,756 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120966e8841567fb439390de6eb507c1c85d_ec3eeaafd032b9a023fe519c0fcbb0c5 is 50, key is test_row_0/A:col10/1733740949571/Put/seqid=0 2024-12-09T10:42:30,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742233_1409 (size=14994) 2024-12-09T10:42:30,760 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:30,764 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120966e8841567fb439390de6eb507c1c85d_ec3eeaafd032b9a023fe519c0fcbb0c5 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120966e8841567fb439390de6eb507c1c85d_ec3eeaafd032b9a023fe519c0fcbb0c5 2024-12-09T10:42:30,765 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/A/5798d21524854dfebc8ac67597b58526, store: [table=TestAcidGuarantees family=A region=ec3eeaafd032b9a023fe519c0fcbb0c5] 2024-12-09T10:42:30,765 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:30,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50954 deadline: 1733741010759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:30,765 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/A/5798d21524854dfebc8ac67597b58526 is 175, key is test_row_0/A:col10/1733740949571/Put/seqid=0 2024-12-09T10:42:30,765 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:30,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50974 deadline: 1733741010759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:30,765 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 7ea43b39fc24,36101,1733740827678 for 'test_row_1' in ec3eeaafd032b9a023fe519c0fcbb0c5 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2198 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T10:42:30,766 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:30,766 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:30,766 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:30,766 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:30,766 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:30,766 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:30,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742234_1410 (size=39949) 2024-12-09T10:42:30,772 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:30,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50962 deadline: 1733741010761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:30,773 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:30,773 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:30,773 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:30,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50928 deadline: 1733741010762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:30,773 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:30,773 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:30,773 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:30,773 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:30,777 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:30,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50942 deadline: 1733741010765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:30,777 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:30,777 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:30,777 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:30,888 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:30,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50962 deadline: 1733741010879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:30,888 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:30,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50928 deadline: 1733741010879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:30,889 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:30,889 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:30,889 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:30,889 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:30,889 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:30,889 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:30,889 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:30,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50942 deadline: 1733741010879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:30,889 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:30,889 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:30,889 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:31,100 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:31,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50962 deadline: 1733741011099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:31,100 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:31,100 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:31,100 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:31,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50928 deadline: 1733741011099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:31,100 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:31,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:31,101 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:31,101 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:31,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50942 deadline: 1733741011099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:31,101 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:31,101 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:31,101 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:31,101 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:31,169 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=329, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/A/5798d21524854dfebc8ac67597b58526 2024-12-09T10:42:31,177 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/B/71b6c8b2385245c3ad62decf88d03f6d is 50, key is test_row_0/B:col10/1733740949571/Put/seqid=0 2024-12-09T10:42:31,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742235_1411 (size=12301) 2024-12-09T10:42:31,418 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:31,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50962 deadline: 1733741011410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:31,418 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:31,419 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:31,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50928 deadline: 1733741011410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:31,419 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:31,419 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:31,419 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:31,419 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:31,419 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:31,419 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:31,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50942 deadline: 1733741011410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:31,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:31,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:31,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:31,588 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/B/71b6c8b2385245c3ad62decf88d03f6d 2024-12-09T10:42:31,597 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/C/0b9c87caa28e418dbfb558e7c5124d64 is 50, key is test_row_0/C:col10/1733740949571/Put/seqid=0 2024-12-09T10:42:31,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-09T10:42:31,600 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T10:42:31,600 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-09T10:42:31,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-12-09T10:42:31,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-09T10:42:31,602 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T10:42:31,603 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T10:42:31,603 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T10:42:31,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742236_1412 (size=12301) 2024-12-09T10:42:31,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-09T10:42:31,756 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-12-09T10:42:31,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. 2024-12-09T10:42:31,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. as already flushing 2024-12-09T10:42:31,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. 2024-12-09T10:42:31,756 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => ec3eeaafd032b9a023fe519c0fcbb0c5, NAME => 'TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:31,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => ec3eeaafd032b9a023fe519c0fcbb0c5, NAME => 'TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:31,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ec3eeaafd032b9a023fe519c0fcbb0c5, NAME => 'TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ec3eeaafd032b9a023fe519c0fcbb0c5, NAME => 'TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:31,781 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:31,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50974 deadline: 1733741011779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:31,782 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 7ea43b39fc24,36101,1733740827678 for 'test_row_2' in ec3eeaafd032b9a023fe519c0fcbb0c5 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2207 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T10:42:31,782 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:31,782 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:31,782 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:31,909 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-12-09T10:42:31,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. 2024-12-09T10:42:31,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. as already flushing 2024-12-09T10:42:31,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. 2024-12-09T10:42:31,909 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => ec3eeaafd032b9a023fe519c0fcbb0c5, NAME => 'TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:31,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => ec3eeaafd032b9a023fe519c0fcbb0c5, NAME => 'TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:31,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ec3eeaafd032b9a023fe519c0fcbb0c5, NAME => 'TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ec3eeaafd032b9a023fe519c0fcbb0c5, NAME => 'TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:31,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-09T10:42:31,938 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:31,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50962 deadline: 1733741011930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:31,939 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:31,939 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:31,939 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:31,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50928 deadline: 1733741011930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:31,939 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:31,939 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:31,939 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:31,939 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:31,939 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:31,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50942 deadline: 1733741011930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:31,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:31,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:31,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:42:32,017 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/C/0b9c87caa28e418dbfb558e7c5124d64 2024-12-09T10:42:32,024 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/A/5798d21524854dfebc8ac67597b58526 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/5798d21524854dfebc8ac67597b58526 2024-12-09T10:42:32,028 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/5798d21524854dfebc8ac67597b58526, entries=200, sequenceid=329, filesize=39.0 K 2024-12-09T10:42:32,028 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/B/71b6c8b2385245c3ad62decf88d03f6d as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B/71b6c8b2385245c3ad62decf88d03f6d 2024-12-09T10:42:32,032 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B/71b6c8b2385245c3ad62decf88d03f6d, entries=150, sequenceid=329, filesize=12.0 K 2024-12-09T10:42:32,033 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/C/0b9c87caa28e418dbfb558e7c5124d64 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C/0b9c87caa28e418dbfb558e7c5124d64 2024-12-09T10:42:32,039 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C/0b9c87caa28e418dbfb558e7c5124d64, entries=150, sequenceid=329, filesize=12.0 K 2024-12-09T10:42:32,040 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~420.80 KB/430896, currentSize=33.54 KB/34350 for ec3eeaafd032b9a023fe519c0fcbb0c5 in 1292ms, sequenceid=329, compaction requested=true 2024-12-09T10:42:32,040 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ec3eeaafd032b9a023fe519c0fcbb0c5: 2024-12-09T10:42:32,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ec3eeaafd032b9a023fe519c0fcbb0c5:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T10:42:32,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:42:32,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ec3eeaafd032b9a023fe519c0fcbb0c5:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T10:42:32,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:42:32,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ec3eeaafd032b9a023fe519c0fcbb0c5:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T10:42:32,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T10:42:32,040 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T10:42:32,041 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T10:42:32,041 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142804 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T10:42:32,041 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49600 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T10:42:32,042 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1541): ec3eeaafd032b9a023fe519c0fcbb0c5/A is initiating minor compaction (all files) 2024-12-09T10:42:32,042 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ec3eeaafd032b9a023fe519c0fcbb0c5/A in TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. 2024-12-09T10:42:32,042 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1541): ec3eeaafd032b9a023fe519c0fcbb0c5/B is initiating minor compaction (all files) 2024-12-09T10:42:32,042 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ec3eeaafd032b9a023fe519c0fcbb0c5/B in TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. 2024-12-09T10:42:32,042 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/078a6e04b2134260a3e6517d8836972e, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/5a8533ef2d7242239c9b0e4d41375dad, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/9e5164d6cc1d42eea48550bd4611704f, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/5798d21524854dfebc8ac67597b58526] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp, totalSize=139.5 K 2024-12-09T10:42:32,042 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. 2024-12-09T10:42:32,042 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. files: [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/078a6e04b2134260a3e6517d8836972e, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/5a8533ef2d7242239c9b0e4d41375dad, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/9e5164d6cc1d42eea48550bd4611704f, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/5798d21524854dfebc8ac67597b58526] 2024-12-09T10:42:32,042 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B/1d382ca18620461d8083d110ba832dc2, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B/0593c55c633a40cfa184846b75cd3395, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B/b2f43342d26d4010b207d658ac001cd0, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B/71b6c8b2385245c3ad62decf88d03f6d] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp, totalSize=48.4 K 2024-12-09T10:42:32,042 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 078a6e04b2134260a3e6517d8836972e, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1733740947150 2024-12-09T10:42:32,042 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d382ca18620461d8083d110ba832dc2, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1733740947150 2024-12-09T10:42:32,042 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a8533ef2d7242239c9b0e4d41375dad, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1733740947284 2024-12-09T10:42:32,042 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 0593c55c633a40cfa184846b75cd3395, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1733740947287 2024-12-09T10:42:32,043 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9e5164d6cc1d42eea48550bd4611704f, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1733740948505 2024-12-09T10:42:32,043 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting b2f43342d26d4010b207d658ac001cd0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1733740948505 2024-12-09T10:42:32,043 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 71b6c8b2385245c3ad62decf88d03f6d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1733740949570 2024-12-09T10:42:32,043 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5798d21524854dfebc8ac67597b58526, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1733740949570 2024-12-09T10:42:32,051 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ec3eeaafd032b9a023fe519c0fcbb0c5#B#compaction#345 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:42:32,052 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/B/3b309163fbb54dab9a09035f2d0b336a is 50, key is test_row_0/B:col10/1733740949571/Put/seqid=0 2024-12-09T10:42:32,053 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ec3eeaafd032b9a023fe519c0fcbb0c5] 2024-12-09T10:42:32,055 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241209526f0ecc1cb94d7583e3f01bc0b425ee_ec3eeaafd032b9a023fe519c0fcbb0c5 store=[table=TestAcidGuarantees family=A region=ec3eeaafd032b9a023fe519c0fcbb0c5] 2024-12-09T10:42:32,058 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241209526f0ecc1cb94d7583e3f01bc0b425ee_ec3eeaafd032b9a023fe519c0fcbb0c5, store=[table=TestAcidGuarantees family=A region=ec3eeaafd032b9a023fe519c0fcbb0c5] 2024-12-09T10:42:32,058 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209526f0ecc1cb94d7583e3f01bc0b425ee_ec3eeaafd032b9a023fe519c0fcbb0c5 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ec3eeaafd032b9a023fe519c0fcbb0c5] 2024-12-09T10:42:32,062 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-12-09T10:42:32,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. 2024-12-09T10:42:32,062 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2902): Flushing ec3eeaafd032b9a023fe519c0fcbb0c5 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-09T10:42:32,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec3eeaafd032b9a023fe519c0fcbb0c5, store=A 2024-12-09T10:42:32,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:42:32,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec3eeaafd032b9a023fe519c0fcbb0c5, store=B 2024-12-09T10:42:32,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:42:32,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec3eeaafd032b9a023fe519c0fcbb0c5, store=C 2024-12-09T10:42:32,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:42:32,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412098e568c80a2264976acacf1d4f2345b47_ec3eeaafd032b9a023fe519c0fcbb0c5 is 50, key is test_row_0/A:col10/1733740950760/Put/seqid=0 2024-12-09T10:42:32,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742238_1414 (size=4469) 2024-12-09T10:42:32,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742237_1413 (size=12439) 2024-12-09T10:42:32,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742239_1415 (size=12454) 2024-12-09T10:42:32,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:32,088 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412098e568c80a2264976acacf1d4f2345b47_ec3eeaafd032b9a023fe519c0fcbb0c5 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412098e568c80a2264976acacf1d4f2345b47_ec3eeaafd032b9a023fe519c0fcbb0c5 2024-12-09T10:42:32,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/A/ca8904e897684452a5d290737d3edc58, store: [table=TestAcidGuarantees family=A region=ec3eeaafd032b9a023fe519c0fcbb0c5] 2024-12-09T10:42:32,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/A/ca8904e897684452a5d290737d3edc58 is 175, key is test_row_0/A:col10/1733740950760/Put/seqid=0 2024-12-09T10:42:32,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742240_1416 (size=31255) 2024-12-09T10:42:32,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-09T10:42:32,246 INFO [Thread-1540 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1540. 2024-12-09T10:42:32,246 DEBUG [Thread-1540 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T10:42:32,246 DEBUG [Thread-1540 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:42:32,247 DEBUG [Thread-1540 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:42:32,247 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T10:42:32,252 INFO [Thread-1534 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1534. 2024-12-09T10:42:32,252 DEBUG [Thread-1534 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T10:42:32,252 DEBUG [Thread-1534 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:42:32,252 DEBUG [Thread-1534 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:42:32,252 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T10:42:32,253 INFO [Thread-1538 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1538. 2024-12-09T10:42:32,253 DEBUG [Thread-1538 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T10:42:32,253 DEBUG [Thread-1538 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:42:32,253 DEBUG [Thread-1538 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:42:32,253 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T10:42:32,255 INFO [Thread-1532 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1532. 2024-12-09T10:42:32,255 DEBUG [Thread-1532 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T10:42:32,255 DEBUG [Thread-1532 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:42:32,255 DEBUG [Thread-1532 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:42:32,255 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T10:42:32,258 INFO [Thread-1536 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1536. 2024-12-09T10:42:32,258 DEBUG [Thread-1536 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T10:42:32,258 DEBUG [Thread-1536 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:42:32,258 DEBUG [Thread-1536 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:42:32,258 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T10:42:32,479 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ec3eeaafd032b9a023fe519c0fcbb0c5#A#compaction#346 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:42:32,480 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/A/5299a6dd7de74f3b8325512460cde3bb is 175, key is test_row_0/A:col10/1733740949571/Put/seqid=0 2024-12-09T10:42:32,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742241_1417 (size=31393) 2024-12-09T10:42:32,486 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/B/3b309163fbb54dab9a09035f2d0b336a as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B/3b309163fbb54dab9a09035f2d0b336a 2024-12-09T10:42:32,490 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ec3eeaafd032b9a023fe519c0fcbb0c5/B of ec3eeaafd032b9a023fe519c0fcbb0c5 into 3b309163fbb54dab9a09035f2d0b336a(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:42:32,490 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ec3eeaafd032b9a023fe519c0fcbb0c5: 2024-12-09T10:42:32,490 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., storeName=ec3eeaafd032b9a023fe519c0fcbb0c5/B, priority=12, startTime=1733740952040; duration=0sec 2024-12-09T10:42:32,490 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T10:42:32,490 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ec3eeaafd032b9a023fe519c0fcbb0c5:B 2024-12-09T10:42:32,490 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T10:42:32,491 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49600 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T10:42:32,491 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1541): ec3eeaafd032b9a023fe519c0fcbb0c5/C is initiating minor compaction (all files) 2024-12-09T10:42:32,491 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ec3eeaafd032b9a023fe519c0fcbb0c5/C in TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. 2024-12-09T10:42:32,491 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C/6dd193f1440e4cd5b76b259169c4c5ef, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C/1f83b2c121824ce5b2eef10ca764bb80, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C/8831859619f144eb8d0264f580a14475, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C/0b9c87caa28e418dbfb558e7c5124d64] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp, totalSize=48.4 K 2024-12-09T10:42:32,492 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 6dd193f1440e4cd5b76b259169c4c5ef, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1733740947150 2024-12-09T10:42:32,492 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f83b2c121824ce5b2eef10ca764bb80, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1733740947287 2024-12-09T10:42:32,492 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 8831859619f144eb8d0264f580a14475, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1733740948505 2024-12-09T10:42:32,492 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b9c87caa28e418dbfb558e7c5124d64, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1733740949570 2024-12-09T10:42:32,495 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=337, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/A/ca8904e897684452a5d290737d3edc58 2024-12-09T10:42:32,499 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ec3eeaafd032b9a023fe519c0fcbb0c5#C#compaction#348 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:42:32,499 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/C/12314537d9d04be59126a9a32ea37ff2 is 50, key is test_row_0/C:col10/1733740949571/Put/seqid=0 2024-12-09T10:42:32,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/B/476350b49fa6436c93fc1080aac7a401 is 50, key is test_row_0/B:col10/1733740950760/Put/seqid=0 2024-12-09T10:42:32,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742242_1418 (size=12439) 2024-12-09T10:42:32,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742243_1419 (size=12301) 2024-12-09T10:42:32,504 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=337 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/B/476350b49fa6436c93fc1080aac7a401 2024-12-09T10:42:32,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/C/fcaadc2e77fd471581fa0b75775da62f is 50, key is test_row_0/C:col10/1733740950760/Put/seqid=0 2024-12-09T10:42:32,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742244_1420 (size=12301) 2024-12-09T10:42:32,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-09T10:42:32,771 INFO [Thread-1523 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1523. 2024-12-09T10:42:32,771 DEBUG [Thread-1523 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T10:42:32,771 DEBUG [Thread-1523 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:42:32,771 DEBUG [Thread-1523 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:42:32,771 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T10:42:32,887 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/A/5299a6dd7de74f3b8325512460cde3bb as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/5299a6dd7de74f3b8325512460cde3bb 2024-12-09T10:42:32,891 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ec3eeaafd032b9a023fe519c0fcbb0c5/A of ec3eeaafd032b9a023fe519c0fcbb0c5 into 5299a6dd7de74f3b8325512460cde3bb(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:42:32,891 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ec3eeaafd032b9a023fe519c0fcbb0c5: 2024-12-09T10:42:32,891 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., storeName=ec3eeaafd032b9a023fe519c0fcbb0c5/A, priority=12, startTime=1733740952040; duration=0sec 2024-12-09T10:42:32,891 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:42:32,891 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ec3eeaafd032b9a023fe519c0fcbb0c5:A 2024-12-09T10:42:32,906 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/C/12314537d9d04be59126a9a32ea37ff2 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C/12314537d9d04be59126a9a32ea37ff2 2024-12-09T10:42:32,910 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ec3eeaafd032b9a023fe519c0fcbb0c5/C of ec3eeaafd032b9a023fe519c0fcbb0c5 into 12314537d9d04be59126a9a32ea37ff2(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:42:32,910 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ec3eeaafd032b9a023fe519c0fcbb0c5: 2024-12-09T10:42:32,910 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., storeName=ec3eeaafd032b9a023fe519c0fcbb0c5/C, priority=12, startTime=1733740952040; duration=0sec 2024-12-09T10:42:32,911 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:42:32,911 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ec3eeaafd032b9a023fe519c0fcbb0c5:C 2024-12-09T10:42:32,913 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=337 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/C/fcaadc2e77fd471581fa0b75775da62f 2024-12-09T10:42:32,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/A/ca8904e897684452a5d290737d3edc58 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/ca8904e897684452a5d290737d3edc58 2024-12-09T10:42:32,919 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/ca8904e897684452a5d290737d3edc58, entries=150, sequenceid=337, filesize=30.5 K 2024-12-09T10:42:32,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/B/476350b49fa6436c93fc1080aac7a401 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B/476350b49fa6436c93fc1080aac7a401 2024-12-09T10:42:32,923 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B/476350b49fa6436c93fc1080aac7a401, entries=150, sequenceid=337, filesize=12.0 K 2024-12-09T10:42:32,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/C/fcaadc2e77fd471581fa0b75775da62f as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C/fcaadc2e77fd471581fa0b75775da62f 2024-12-09T10:42:32,926 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C/fcaadc2e77fd471581fa0b75775da62f, entries=150, sequenceid=337, filesize=12.0 K 2024-12-09T10:42:32,927 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=6.71 KB/6870 for ec3eeaafd032b9a023fe519c0fcbb0c5 in 865ms, sequenceid=337, compaction requested=false 2024-12-09T10:42:32,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for ec3eeaafd032b9a023fe519c0fcbb0c5: 2024-12-09T10:42:32,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. 2024-12-09T10:42:32,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-12-09T10:42:32,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-12-09T10:42:32,930 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-12-09T10:42:32,930 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3260 sec 2024-12-09T10:42:32,932 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 1.3310 sec 2024-12-09T10:42:32,951 INFO [Thread-1527 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1527. 2024-12-09T10:42:32,951 DEBUG [Thread-1527 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T10:42:32,951 DEBUG [Thread-1527 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:42:32,951 INFO [Thread-1529 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1529. 2024-12-09T10:42:32,951 DEBUG [Thread-1527 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:42:32,951 DEBUG [Thread-1529 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T10:42:32,951 DEBUG [Thread-1529 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:42:32,951 INFO [Thread-1521 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1521. 2024-12-09T10:42:32,951 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T10:42:32,951 DEBUG [Thread-1521 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T10:42:32,951 DEBUG [Thread-1529 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:42:32,951 DEBUG [Thread-1521 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:42:32,952 DEBUG [Thread-1521 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:42:32,952 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T10:42:32,952 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T10:42:33,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-09T10:42:33,750 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T10:42:33,800 INFO [Thread-1525 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1525. 2024-12-09T10:42:33,800 DEBUG [Thread-1525 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T10:42:33,801 DEBUG [Thread-1525 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:42:33,801 DEBUG [Thread-1525 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:42:33,801 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T10:42:33,801 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-09T10:42:33,801 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 49 2024-12-09T10:42:33,801 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 57 2024-12-09T10:42:33,801 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 43 2024-12-09T10:42:33,801 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 65 2024-12-09T10:42:33,801 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 61 2024-12-09T10:42:33,801 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-09T10:42:33,801 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-09T10:42:33,801 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 886 2024-12-09T10:42:33,801 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2658 rows 2024-12-09T10:42:33,801 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 889 2024-12-09T10:42:33,801 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2665 rows 2024-12-09T10:42:33,801 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 890 2024-12-09T10:42:33,801 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2665 rows 2024-12-09T10:42:33,801 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 886 2024-12-09T10:42:33,801 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2656 rows 2024-12-09T10:42:33,801 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 890 2024-12-09T10:42:33,801 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2668 rows 2024-12-09T10:42:33,801 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-09T10:42:33,801 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T10:42:33,801 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:42:33,802 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:42:33,802 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T10:42:33,802 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-12-09T10:42:33,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-09T10:42:33,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-09T10:42:33,805 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733740953805"}]},"ts":"1733740953805"} 2024-12-09T10:42:33,806 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-09T10:42:33,806 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-09T10:42:33,807 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-09T10:42:33,808 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ec3eeaafd032b9a023fe519c0fcbb0c5, UNASSIGN}] 2024-12-09T10:42:33,808 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ec3eeaafd032b9a023fe519c0fcbb0c5, UNASSIGN 2024-12-09T10:42:33,809 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=115 updating hbase:meta row=ec3eeaafd032b9a023fe519c0fcbb0c5, regionState=CLOSING, regionLocation=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:33,810 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ec3eeaafd032b9a023fe519c0fcbb0c5, UNASSIGN because future has completed 2024-12-09T10:42:33,810 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-09T10:42:33,810 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; CloseRegionProcedure ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678}] 2024-12-09T10:42:33,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-09T10:42:33,962 INFO [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] handler.UnassignRegionHandler(122): Close ec3eeaafd032b9a023fe519c0fcbb0c5 2024-12-09T10:42:33,962 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-09T10:42:33,962 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1722): Closing ec3eeaafd032b9a023fe519c0fcbb0c5, disabling compactions & flushes 2024-12-09T10:42:33,962 INFO [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. 2024-12-09T10:42:33,962 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. 2024-12-09T10:42:33,962 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. after waiting 0 ms 2024-12-09T10:42:33,962 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. 2024-12-09T10:42:33,963 INFO [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(2902): Flushing ec3eeaafd032b9a023fe519c0fcbb0c5 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-09T10:42:33,963 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec3eeaafd032b9a023fe519c0fcbb0c5, store=A 2024-12-09T10:42:33,963 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:42:33,963 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec3eeaafd032b9a023fe519c0fcbb0c5, store=B 2024-12-09T10:42:33,963 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:42:33,963 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec3eeaafd032b9a023fe519c0fcbb0c5, store=C 2024-12-09T10:42:33,963 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:42:33,968 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209b8d4594b3b6341c9bc2b73aa2d7d0749_ec3eeaafd032b9a023fe519c0fcbb0c5 is 50, key is test_row_0/A:col10/1733740952950/Put/seqid=0 2024-12-09T10:42:33,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742245_1421 (size=12454) 2024-12-09T10:42:34,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-09T10:42:34,372 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:34,375 INFO [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209b8d4594b3b6341c9bc2b73aa2d7d0749_ec3eeaafd032b9a023fe519c0fcbb0c5 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209b8d4594b3b6341c9bc2b73aa2d7d0749_ec3eeaafd032b9a023fe519c0fcbb0c5 2024-12-09T10:42:34,376 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/A/a1084a544e1a45e284a74b5f14673a60, store: [table=TestAcidGuarantees family=A region=ec3eeaafd032b9a023fe519c0fcbb0c5] 2024-12-09T10:42:34,376 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/A/a1084a544e1a45e284a74b5f14673a60 is 175, key is test_row_0/A:col10/1733740952950/Put/seqid=0 2024-12-09T10:42:34,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742246_1422 (size=31255) 2024-12-09T10:42:34,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-09T10:42:34,780 INFO [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=348, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/A/a1084a544e1a45e284a74b5f14673a60 2024-12-09T10:42:34,789 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/B/f0f004d1b14848e2b272fa4e59d52a70 is 50, key is test_row_0/B:col10/1733740952950/Put/seqid=0 2024-12-09T10:42:34,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742247_1423 (size=12301) 2024-12-09T10:42:34,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-09T10:42:35,212 INFO [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=348 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/B/f0f004d1b14848e2b272fa4e59d52a70 2024-12-09T10:42:35,218 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/C/7ab30763282c4dc2b9e6828faf96a305 is 50, key is test_row_0/C:col10/1733740952950/Put/seqid=0 2024-12-09T10:42:35,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742248_1424 (size=12301) 2024-12-09T10:42:35,621 INFO [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=348 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/C/7ab30763282c4dc2b9e6828faf96a305 2024-12-09T10:42:35,625 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/A/a1084a544e1a45e284a74b5f14673a60 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/a1084a544e1a45e284a74b5f14673a60 2024-12-09T10:42:35,628 INFO [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/a1084a544e1a45e284a74b5f14673a60, entries=150, sequenceid=348, filesize=30.5 K 2024-12-09T10:42:35,629 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/B/f0f004d1b14848e2b272fa4e59d52a70 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B/f0f004d1b14848e2b272fa4e59d52a70 2024-12-09T10:42:35,632 INFO [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B/f0f004d1b14848e2b272fa4e59d52a70, entries=150, sequenceid=348, filesize=12.0 K 2024-12-09T10:42:35,632 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/.tmp/C/7ab30763282c4dc2b9e6828faf96a305 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C/7ab30763282c4dc2b9e6828faf96a305 2024-12-09T10:42:35,635 INFO [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C/7ab30763282c4dc2b9e6828faf96a305, entries=150, sequenceid=348, filesize=12.0 K 2024-12-09T10:42:35,636 INFO [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for ec3eeaafd032b9a023fe519c0fcbb0c5 in 1674ms, sequenceid=348, compaction requested=true 2024-12-09T10:42:35,636 DEBUG [StoreCloser-TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/078a6e04b2134260a3e6517d8836972e, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/5a8533ef2d7242239c9b0e4d41375dad, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/9e5164d6cc1d42eea48550bd4611704f, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/5798d21524854dfebc8ac67597b58526] to archive 2024-12-09T10:42:35,637 DEBUG [StoreCloser-TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T10:42:35,638 DEBUG [StoreCloser-TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/078a6e04b2134260a3e6517d8836972e to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/078a6e04b2134260a3e6517d8836972e 2024-12-09T10:42:35,639 DEBUG [StoreCloser-TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/5a8533ef2d7242239c9b0e4d41375dad to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/5a8533ef2d7242239c9b0e4d41375dad 2024-12-09T10:42:35,640 DEBUG [StoreCloser-TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/9e5164d6cc1d42eea48550bd4611704f to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/9e5164d6cc1d42eea48550bd4611704f 2024-12-09T10:42:35,641 DEBUG [StoreCloser-TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/5798d21524854dfebc8ac67597b58526 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/5798d21524854dfebc8ac67597b58526 2024-12-09T10:42:35,642 DEBUG [StoreCloser-TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B/1d382ca18620461d8083d110ba832dc2, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B/0593c55c633a40cfa184846b75cd3395, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B/b2f43342d26d4010b207d658ac001cd0, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B/71b6c8b2385245c3ad62decf88d03f6d] to archive 2024-12-09T10:42:35,642 DEBUG [StoreCloser-TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T10:42:35,643 DEBUG [StoreCloser-TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B/1d382ca18620461d8083d110ba832dc2 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B/1d382ca18620461d8083d110ba832dc2 2024-12-09T10:42:35,644 DEBUG [StoreCloser-TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B/0593c55c633a40cfa184846b75cd3395 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B/0593c55c633a40cfa184846b75cd3395 2024-12-09T10:42:35,645 DEBUG [StoreCloser-TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B/b2f43342d26d4010b207d658ac001cd0 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B/b2f43342d26d4010b207d658ac001cd0 2024-12-09T10:42:35,645 DEBUG [StoreCloser-TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B/71b6c8b2385245c3ad62decf88d03f6d to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B/71b6c8b2385245c3ad62decf88d03f6d 2024-12-09T10:42:35,646 DEBUG [StoreCloser-TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C/6dd193f1440e4cd5b76b259169c4c5ef, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C/1f83b2c121824ce5b2eef10ca764bb80, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C/8831859619f144eb8d0264f580a14475, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C/0b9c87caa28e418dbfb558e7c5124d64] to archive 2024-12-09T10:42:35,647 DEBUG [StoreCloser-TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T10:42:35,648 DEBUG [StoreCloser-TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C/6dd193f1440e4cd5b76b259169c4c5ef to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C/6dd193f1440e4cd5b76b259169c4c5ef 2024-12-09T10:42:35,649 DEBUG [StoreCloser-TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C/1f83b2c121824ce5b2eef10ca764bb80 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C/1f83b2c121824ce5b2eef10ca764bb80 2024-12-09T10:42:35,649 DEBUG [StoreCloser-TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C/8831859619f144eb8d0264f580a14475 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C/8831859619f144eb8d0264f580a14475 2024-12-09T10:42:35,650 DEBUG [StoreCloser-TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C/0b9c87caa28e418dbfb558e7c5124d64 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C/0b9c87caa28e418dbfb558e7c5124d64 2024-12-09T10:42:35,653 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/recovered.edits/351.seqid, newMaxSeqId=351, maxSeqId=4 2024-12-09T10:42:35,654 INFO [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. 2024-12-09T10:42:35,654 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1676): Region close journal for ec3eeaafd032b9a023fe519c0fcbb0c5: Waiting for close lock at 1733740953962Running coprocessor pre-close hooks at 1733740953962Disabling compacts and flushes for region at 1733740953962Disabling writes for close at 1733740953962Obtaining lock to block concurrent updates at 1733740953963 (+1 ms)Preparing flush snapshotting stores in ec3eeaafd032b9a023fe519c0fcbb0c5 at 1733740953963Finished memstore snapshotting TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1733740953963Flushing stores of TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5. at 1733740953963Flushing ec3eeaafd032b9a023fe519c0fcbb0c5/A: creating writer at 1733740953963Flushing mob file ec3eeaafd032b9a023fe519c0fcbb0c5/A: appending metadata at 1733740953968 (+5 ms)Flushing mob file ec3eeaafd032b9a023fe519c0fcbb0c5/A: closing flushed file at 1733740953968Flushing ec3eeaafd032b9a023fe519c0fcbb0c5/A: appending metadata at 1733740954376 (+408 ms)Flushing ec3eeaafd032b9a023fe519c0fcbb0c5/A: closing flushed file at 1733740954376Flushing ec3eeaafd032b9a023fe519c0fcbb0c5/B: creating writer at 1733740954786 (+410 ms)Flushing ec3eeaafd032b9a023fe519c0fcbb0c5/B: appending metadata at 1733740954788 (+2 ms)Flushing ec3eeaafd032b9a023fe519c0fcbb0c5/B: closing flushed file at 1733740954788Flushing ec3eeaafd032b9a023fe519c0fcbb0c5/C: creating writer at 1733740955215 (+427 ms)Flushing ec3eeaafd032b9a023fe519c0fcbb0c5/C: appending metadata at 1733740955217 (+2 ms)Flushing ec3eeaafd032b9a023fe519c0fcbb0c5/C: closing flushed file at 1733740955217Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6d006578: reopening flushed file at 1733740955625 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3e6be94d: reopening flushed file at 1733740955629 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5f309351: reopening flushed file at 1733740955632 (+3 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for ec3eeaafd032b9a023fe519c0fcbb0c5 in 1674ms, sequenceid=348, compaction requested=true at 1733740955636 (+4 ms)Writing region close event to WAL at 1733740955651 (+15 ms)Running coprocessor post-close hooks at 1733740955654 (+3 ms)Closed at 1733740955654 2024-12-09T10:42:35,656 INFO [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] handler.UnassignRegionHandler(157): Closed ec3eeaafd032b9a023fe519c0fcbb0c5 2024-12-09T10:42:35,656 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=115 updating hbase:meta row=ec3eeaafd032b9a023fe519c0fcbb0c5, regionState=CLOSED 2024-12-09T10:42:35,657 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=116, ppid=115, state=RUNNABLE, hasLock=false; CloseRegionProcedure ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 because future has completed 2024-12-09T10:42:35,659 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-12-09T10:42:35,659 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; CloseRegionProcedure ec3eeaafd032b9a023fe519c0fcbb0c5, server=7ea43b39fc24,36101,1733740827678 in 1.8480 sec 2024-12-09T10:42:35,661 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=115, resume processing ppid=114 2024-12-09T10:42:35,661 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, ppid=114, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ec3eeaafd032b9a023fe519c0fcbb0c5, UNASSIGN in 1.8510 sec 2024-12-09T10:42:35,663 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-12-09T10:42:35,663 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8550 sec 2024-12-09T10:42:35,664 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733740955664"}]},"ts":"1733740955664"} 2024-12-09T10:42:35,665 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-09T10:42:35,665 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-09T10:42:35,667 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8640 sec 2024-12-09T10:42:35,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-09T10:42:35,949 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-09T10:42:35,950 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-12-09T10:42:35,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T10:42:35,952 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=117, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T10:42:35,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-09T10:42:35,952 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=117, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T10:42:35,954 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5 2024-12-09T10:42:35,955 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A, FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B, FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C, FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/recovered.edits] 2024-12-09T10:42:35,958 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/5299a6dd7de74f3b8325512460cde3bb to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/5299a6dd7de74f3b8325512460cde3bb 2024-12-09T10:42:35,958 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/a1084a544e1a45e284a74b5f14673a60 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/a1084a544e1a45e284a74b5f14673a60 2024-12-09T10:42:35,959 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/ca8904e897684452a5d290737d3edc58 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/A/ca8904e897684452a5d290737d3edc58 2024-12-09T10:42:35,961 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B/3b309163fbb54dab9a09035f2d0b336a to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B/3b309163fbb54dab9a09035f2d0b336a 2024-12-09T10:42:35,962 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B/476350b49fa6436c93fc1080aac7a401 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B/476350b49fa6436c93fc1080aac7a401 2024-12-09T10:42:35,963 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B/f0f004d1b14848e2b272fa4e59d52a70 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/B/f0f004d1b14848e2b272fa4e59d52a70 2024-12-09T10:42:35,964 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C/12314537d9d04be59126a9a32ea37ff2 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C/12314537d9d04be59126a9a32ea37ff2 2024-12-09T10:42:35,966 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C/7ab30763282c4dc2b9e6828faf96a305 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C/7ab30763282c4dc2b9e6828faf96a305 2024-12-09T10:42:35,967 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C/fcaadc2e77fd471581fa0b75775da62f to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/C/fcaadc2e77fd471581fa0b75775da62f 2024-12-09T10:42:35,969 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/recovered.edits/351.seqid to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5/recovered.edits/351.seqid 2024-12-09T10:42:35,969 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/ec3eeaafd032b9a023fe519c0fcbb0c5 2024-12-09T10:42:35,969 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-09T10:42:35,970 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-09T10:42:35,970 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-09T10:42:35,973 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412091b691da6f0b64aed9752e6dd681b62c2_ec3eeaafd032b9a023fe519c0fcbb0c5 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412091b691da6f0b64aed9752e6dd681b62c2_ec3eeaafd032b9a023fe519c0fcbb0c5 2024-12-09T10:42:35,974 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412091db9286d29694c74ba342e6c9be3bcd5_ec3eeaafd032b9a023fe519c0fcbb0c5 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412091db9286d29694c74ba342e6c9be3bcd5_ec3eeaafd032b9a023fe519c0fcbb0c5 2024-12-09T10:42:35,975 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209200df86faf4246319f3e78467a004d05_ec3eeaafd032b9a023fe519c0fcbb0c5 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209200df86faf4246319f3e78467a004d05_ec3eeaafd032b9a023fe519c0fcbb0c5 2024-12-09T10:42:35,975 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412092075e703a46b405caf0aa82842eea445_ec3eeaafd032b9a023fe519c0fcbb0c5 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412092075e703a46b405caf0aa82842eea445_ec3eeaafd032b9a023fe519c0fcbb0c5 2024-12-09T10:42:35,976 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120941cdda7319904ec6aacb431c5608c7af_ec3eeaafd032b9a023fe519c0fcbb0c5 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120941cdda7319904ec6aacb431c5608c7af_ec3eeaafd032b9a023fe519c0fcbb0c5 2024-12-09T10:42:35,977 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120959a48fa660544664b44e8f222406b1dd_ec3eeaafd032b9a023fe519c0fcbb0c5 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120959a48fa660544664b44e8f222406b1dd_ec3eeaafd032b9a023fe519c0fcbb0c5 2024-12-09T10:42:35,978 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412095d49d24868ac4821bf5688478bb3b62d_ec3eeaafd032b9a023fe519c0fcbb0c5 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412095d49d24868ac4821bf5688478bb3b62d_ec3eeaafd032b9a023fe519c0fcbb0c5 2024-12-09T10:42:35,979 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412096048e11995ec4bdf871b146a9a41a78d_ec3eeaafd032b9a023fe519c0fcbb0c5 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412096048e11995ec4bdf871b146a9a41a78d_ec3eeaafd032b9a023fe519c0fcbb0c5 2024-12-09T10:42:35,980 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120966e8841567fb439390de6eb507c1c85d_ec3eeaafd032b9a023fe519c0fcbb0c5 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120966e8841567fb439390de6eb507c1c85d_ec3eeaafd032b9a023fe519c0fcbb0c5 2024-12-09T10:42:35,981 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412097b2c82030b3748e8a16895b22263069b_ec3eeaafd032b9a023fe519c0fcbb0c5 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412097b2c82030b3748e8a16895b22263069b_ec3eeaafd032b9a023fe519c0fcbb0c5 2024-12-09T10:42:35,982 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412098e568c80a2264976acacf1d4f2345b47_ec3eeaafd032b9a023fe519c0fcbb0c5 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412098e568c80a2264976acacf1d4f2345b47_ec3eeaafd032b9a023fe519c0fcbb0c5 2024-12-09T10:42:35,983 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209b3721b0eedb04ed99a72186a0a27ef40_ec3eeaafd032b9a023fe519c0fcbb0c5 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209b3721b0eedb04ed99a72186a0a27ef40_ec3eeaafd032b9a023fe519c0fcbb0c5 2024-12-09T10:42:35,984 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209b8d4594b3b6341c9bc2b73aa2d7d0749_ec3eeaafd032b9a023fe519c0fcbb0c5 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209b8d4594b3b6341c9bc2b73aa2d7d0749_ec3eeaafd032b9a023fe519c0fcbb0c5 2024-12-09T10:42:35,984 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209b8fc284b36ec421ba3bd323e296424ac_ec3eeaafd032b9a023fe519c0fcbb0c5 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209b8fc284b36ec421ba3bd323e296424ac_ec3eeaafd032b9a023fe519c0fcbb0c5 2024-12-09T10:42:35,985 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209ba989f47e68e4864a5756e018f1529bc_ec3eeaafd032b9a023fe519c0fcbb0c5 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209ba989f47e68e4864a5756e018f1529bc_ec3eeaafd032b9a023fe519c0fcbb0c5 2024-12-09T10:42:35,986 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209bdc627d5ab7c4749bc3d2ff9af0b7f6c_ec3eeaafd032b9a023fe519c0fcbb0c5 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209bdc627d5ab7c4749bc3d2ff9af0b7f6c_ec3eeaafd032b9a023fe519c0fcbb0c5 2024-12-09T10:42:35,987 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209c7929d69133f43c791344dbac64a1116_ec3eeaafd032b9a023fe519c0fcbb0c5 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209c7929d69133f43c791344dbac64a1116_ec3eeaafd032b9a023fe519c0fcbb0c5 2024-12-09T10:42:35,988 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209e3d140fb687e4205a5c4d4e62ee047ad_ec3eeaafd032b9a023fe519c0fcbb0c5 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209e3d140fb687e4205a5c4d4e62ee047ad_ec3eeaafd032b9a023fe519c0fcbb0c5 2024-12-09T10:42:35,988 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-09T10:42:35,990 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=117, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T10:42:35,992 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-09T10:42:35,993 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-09T10:42:35,994 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=117, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T10:42:35,994 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-09T10:42:35,995 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733740955994"}]},"ts":"9223372036854775807"} 2024-12-09T10:42:35,996 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-09T10:42:35,996 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => ec3eeaafd032b9a023fe519c0fcbb0c5, NAME => 'TestAcidGuarantees,,1733740929131.ec3eeaafd032b9a023fe519c0fcbb0c5.', STARTKEY => '', ENDKEY => ''}] 2024-12-09T10:42:35,996 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-09T10:42:35,996 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733740955996"}]},"ts":"9223372036854775807"} 2024-12-09T10:42:35,998 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-09T10:42:35,998 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=117, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T10:42:35,999 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 48 msec 2024-12-09T10:42:36,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-09T10:42:36,059 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-09T10:42:36,059 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-09T10:42:36,069 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=252 (was 246) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-11 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1626372997_22 at /127.0.0.1:35960 [Waiting for operation #204] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_COMPACTED_FILES_DISCHARGER-regionserver/7ea43b39fc24:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1626372997_22 at /127.0.0.1:35948 [Waiting for operation #201] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_421252992_22 at /127.0.0.1:43632 [Waiting for operation #472] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-10 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_421252992_22 at /127.0.0.1:35964 [Waiting for operation #187] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=459 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=351 (was 328) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7361 (was 7894) 2024-12-09T10:42:36,078 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=252, OpenFileDescriptor=459, MaxFileDescriptor=1048576, SystemLoadAverage=351, ProcessCount=11, AvailableMemoryMB=7361 2024-12-09T10:42:36,079 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-09T10:42:36,079 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T10:42:36,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] procedure2.ProcedureExecutor(1139): Stored pid=118, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-09T10:42:36,081 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-09T10:42:36,081 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:36,081 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 118 2024-12-09T10:42:36,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-12-09T10:42:36,082 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-09T10:42:36,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742249_1425 (size=1061) 2024-12-09T10:42:36,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-12-09T10:42:36,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-12-09T10:42:36,488 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160 2024-12-09T10:42:36,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742250_1426 (size=53) 2024-12-09T10:42:36,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-12-09T10:42:36,893 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T10:42:36,894 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing a9d4962ae8c180b5b7fa587448872d04, disabling compactions & flushes 2024-12-09T10:42:36,894 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. 2024-12-09T10:42:36,894 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. 2024-12-09T10:42:36,894 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. after waiting 0 ms 2024-12-09T10:42:36,894 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. 2024-12-09T10:42:36,894 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. 2024-12-09T10:42:36,894 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for a9d4962ae8c180b5b7fa587448872d04: Waiting for close lock at 1733740956894Disabling compacts and flushes for region at 1733740956894Disabling writes for close at 1733740956894Writing region close event to WAL at 1733740956894Closed at 1733740956894 2024-12-09T10:42:36,895 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-09T10:42:36,895 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733740956895"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733740956895"}]},"ts":"1733740956895"} 2024-12-09T10:42:36,896 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-09T10:42:36,897 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-09T10:42:36,897 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733740956897"}]},"ts":"1733740956897"} 2024-12-09T10:42:36,898 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-09T10:42:36,899 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a9d4962ae8c180b5b7fa587448872d04, ASSIGN}] 2024-12-09T10:42:36,900 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a9d4962ae8c180b5b7fa587448872d04, ASSIGN 2024-12-09T10:42:36,900 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a9d4962ae8c180b5b7fa587448872d04, ASSIGN; state=OFFLINE, location=7ea43b39fc24,36101,1733740827678; forceNewPlan=false, retain=false 2024-12-09T10:42:37,051 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=119 updating hbase:meta row=a9d4962ae8c180b5b7fa587448872d04, regionState=OPENING, regionLocation=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:37,052 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a9d4962ae8c180b5b7fa587448872d04, ASSIGN because future has completed 2024-12-09T10:42:37,052 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; OpenRegionProcedure a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678}] 2024-12-09T10:42:37,207 INFO [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. 2024-12-09T10:42:37,207 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(7752): Opening region: {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} 2024-12-09T10:42:37,207 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees a9d4962ae8c180b5b7fa587448872d04 2024-12-09T10:42:37,207 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T10:42:37,207 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(7794): checking encryption for a9d4962ae8c180b5b7fa587448872d04 2024-12-09T10:42:37,207 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(7797): checking classloading for a9d4962ae8c180b5b7fa587448872d04 2024-12-09T10:42:37,209 INFO [StoreOpener-a9d4962ae8c180b5b7fa587448872d04-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region a9d4962ae8c180b5b7fa587448872d04 2024-12-09T10:42:37,210 INFO [StoreOpener-a9d4962ae8c180b5b7fa587448872d04-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T10:42:37,210 INFO [StoreOpener-a9d4962ae8c180b5b7fa587448872d04-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a9d4962ae8c180b5b7fa587448872d04 columnFamilyName A 2024-12-09T10:42:37,210 DEBUG [StoreOpener-a9d4962ae8c180b5b7fa587448872d04-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:37,210 INFO [StoreOpener-a9d4962ae8c180b5b7fa587448872d04-1 {}] regionserver.HStore(327): Store=a9d4962ae8c180b5b7fa587448872d04/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T10:42:37,210 INFO [StoreOpener-a9d4962ae8c180b5b7fa587448872d04-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region a9d4962ae8c180b5b7fa587448872d04 2024-12-09T10:42:37,211 INFO [StoreOpener-a9d4962ae8c180b5b7fa587448872d04-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T10:42:37,211 INFO [StoreOpener-a9d4962ae8c180b5b7fa587448872d04-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a9d4962ae8c180b5b7fa587448872d04 columnFamilyName B 2024-12-09T10:42:37,211 DEBUG [StoreOpener-a9d4962ae8c180b5b7fa587448872d04-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:37,212 INFO [StoreOpener-a9d4962ae8c180b5b7fa587448872d04-1 {}] regionserver.HStore(327): Store=a9d4962ae8c180b5b7fa587448872d04/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T10:42:37,212 INFO [StoreOpener-a9d4962ae8c180b5b7fa587448872d04-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region a9d4962ae8c180b5b7fa587448872d04 2024-12-09T10:42:37,213 INFO [StoreOpener-a9d4962ae8c180b5b7fa587448872d04-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T10:42:37,213 INFO [StoreOpener-a9d4962ae8c180b5b7fa587448872d04-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a9d4962ae8c180b5b7fa587448872d04 columnFamilyName C 2024-12-09T10:42:37,213 DEBUG [StoreOpener-a9d4962ae8c180b5b7fa587448872d04-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:42:37,213 INFO [StoreOpener-a9d4962ae8c180b5b7fa587448872d04-1 {}] regionserver.HStore(327): Store=a9d4962ae8c180b5b7fa587448872d04/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T10:42:37,213 INFO [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. 2024-12-09T10:42:37,213 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1038): replaying wal for a9d4962ae8c180b5b7fa587448872d04 2024-12-09T10:42:37,214 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04 2024-12-09T10:42:37,214 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04 2024-12-09T10:42:37,215 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1048): stopping wal replay for a9d4962ae8c180b5b7fa587448872d04 2024-12-09T10:42:37,215 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1060): Cleaning up temporary data for a9d4962ae8c180b5b7fa587448872d04 2024-12-09T10:42:37,215 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-09T10:42:37,216 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1093): writing seq id for a9d4962ae8c180b5b7fa587448872d04 2024-12-09T10:42:37,217 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T10:42:37,217 INFO [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1114): Opened a9d4962ae8c180b5b7fa587448872d04; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62905143, jitterRate=-0.06264032423496246}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-09T10:42:37,217 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a9d4962ae8c180b5b7fa587448872d04 2024-12-09T10:42:37,218 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1006): Region open journal for a9d4962ae8c180b5b7fa587448872d04: Running coprocessor pre-open hook at 1733740957208Writing region info on filesystem at 1733740957208Initializing all the Stores at 1733740957208Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733740957208Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733740957208Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733740957208Cleaning up temporary data from old regions at 1733740957215 (+7 ms)Running coprocessor post-open hooks at 1733740957217 (+2 ms)Region opened successfully at 1733740957218 (+1 ms) 2024-12-09T10:42:37,219 INFO [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., pid=120, masterSystemTime=1733740957204 2024-12-09T10:42:37,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-12-09T10:42:37,220 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. 2024-12-09T10:42:37,220 INFO [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. 2024-12-09T10:42:37,221 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=119 updating hbase:meta row=a9d4962ae8c180b5b7fa587448872d04, regionState=OPEN, openSeqNum=2, regionLocation=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:37,222 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=120, ppid=119, state=RUNNABLE, hasLock=false; OpenRegionProcedure a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 because future has completed 2024-12-09T10:42:37,224 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-12-09T10:42:37,224 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; OpenRegionProcedure a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 in 170 msec 2024-12-09T10:42:37,225 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=119, resume processing ppid=118 2024-12-09T10:42:37,225 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, ppid=118, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a9d4962ae8c180b5b7fa587448872d04, ASSIGN in 325 msec 2024-12-09T10:42:37,226 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-09T10:42:37,226 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733740957226"}]},"ts":"1733740957226"} 2024-12-09T10:42:37,227 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-09T10:42:37,227 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-09T10:42:37,229 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1490 sec 2024-12-09T10:42:38,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-12-09T10:42:38,229 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-09T10:42:38,231 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@106341c8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:42:38,231 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ea43b39fc24,35095,-1 for getting cluster id 2024-12-09T10:42:38,231 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T10:42:38,232 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0945b5ae-b46b-4b8f-af48-56b0bb92df09' 2024-12-09T10:42:38,232 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T10:42:38,232 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0945b5ae-b46b-4b8f-af48-56b0bb92df09" 2024-12-09T10:42:38,232 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a00e980, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:42:38,232 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ea43b39fc24,35095,-1] 2024-12-09T10:42:38,232 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T10:42:38,232 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:42:38,233 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47584, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T10:42:38,234 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8538dee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:42:38,234 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T10:42:38,235 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ea43b39fc24,36101,1733740827678, seqNum=-1] 2024-12-09T10:42:38,235 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T10:42:38,236 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59104, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T10:42:38,237 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-09T10:42:38,237 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 7ea43b39fc24,35095,1733740826918 2024-12-09T10:42:38,237 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@26d10090 2024-12-09T10:42:38,237 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-09T10:42:38,238 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47586, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-09T10:42:38,239 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6821edd8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:42:38,239 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ea43b39fc24,35095,-1 for getting cluster id 2024-12-09T10:42:38,239 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T10:42:38,240 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0945b5ae-b46b-4b8f-af48-56b0bb92df09' 2024-12-09T10:42:38,240 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T10:42:38,240 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0945b5ae-b46b-4b8f-af48-56b0bb92df09" 2024-12-09T10:42:38,240 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74ac8314, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:42:38,240 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ea43b39fc24,35095,-1] 2024-12-09T10:42:38,240 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T10:42:38,240 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:42:38,241 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47606, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T10:42:38,241 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b8a0917, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:42:38,242 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42981879, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:42:38,242 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ea43b39fc24,35095,-1 for getting cluster id 2024-12-09T10:42:38,242 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T10:42:38,243 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0945b5ae-b46b-4b8f-af48-56b0bb92df09' 2024-12-09T10:42:38,243 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T10:42:38,243 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0945b5ae-b46b-4b8f-af48-56b0bb92df09" 2024-12-09T10:42:38,243 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55f2c4c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:42:38,243 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ea43b39fc24,35095,-1] 2024-12-09T10:42:38,243 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T10:42:38,244 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:42:38,244 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47622, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T10:42:38,245 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@100d48a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:42:38,245 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@736bcb1a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:42:38,245 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ea43b39fc24,35095,-1 for getting cluster id 2024-12-09T10:42:38,245 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T10:42:38,246 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0945b5ae-b46b-4b8f-af48-56b0bb92df09' 2024-12-09T10:42:38,246 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T10:42:38,246 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0945b5ae-b46b-4b8f-af48-56b0bb92df09" 2024-12-09T10:42:38,246 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@938508a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:42:38,246 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ea43b39fc24,35095,-1] 2024-12-09T10:42:38,246 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T10:42:38,246 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:42:38,247 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47644, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T10:42:38,247 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@582a95b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:42:38,248 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74035f7d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:42:38,248 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ea43b39fc24,35095,-1 for getting cluster id 2024-12-09T10:42:38,248 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T10:42:38,248 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0945b5ae-b46b-4b8f-af48-56b0bb92df09' 2024-12-09T10:42:38,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T10:42:38,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0945b5ae-b46b-4b8f-af48-56b0bb92df09" 2024-12-09T10:42:38,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49307822, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:42:38,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ea43b39fc24,35095,-1] 2024-12-09T10:42:38,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T10:42:38,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:42:38,250 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47658, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T10:42:38,250 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24b09b1e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:42:38,251 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d742ef2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:42:38,251 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ea43b39fc24,35095,-1 for getting cluster id 2024-12-09T10:42:38,251 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T10:42:38,251 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0945b5ae-b46b-4b8f-af48-56b0bb92df09' 2024-12-09T10:42:38,251 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T10:42:38,252 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0945b5ae-b46b-4b8f-af48-56b0bb92df09" 2024-12-09T10:42:38,252 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3dffd8fe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:42:38,252 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ea43b39fc24,35095,-1] 2024-12-09T10:42:38,252 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T10:42:38,252 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:42:38,252 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47678, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T10:42:38,253 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62829b3e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:42:38,254 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f1fe171, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:42:38,254 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ea43b39fc24,35095,-1 for getting cluster id 2024-12-09T10:42:38,254 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T10:42:38,254 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0945b5ae-b46b-4b8f-af48-56b0bb92df09' 2024-12-09T10:42:38,254 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T10:42:38,255 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0945b5ae-b46b-4b8f-af48-56b0bb92df09" 2024-12-09T10:42:38,255 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67671b91, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:42:38,255 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ea43b39fc24,35095,-1] 2024-12-09T10:42:38,255 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T10:42:38,255 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:42:38,256 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47702, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T10:42:38,256 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@169df612, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:42:38,257 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75a383c2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:42:38,257 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ea43b39fc24,35095,-1 for getting cluster id 2024-12-09T10:42:38,257 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T10:42:38,257 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0945b5ae-b46b-4b8f-af48-56b0bb92df09' 2024-12-09T10:42:38,258 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T10:42:38,258 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0945b5ae-b46b-4b8f-af48-56b0bb92df09" 2024-12-09T10:42:38,258 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ee52064, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:42:38,258 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ea43b39fc24,35095,-1] 2024-12-09T10:42:38,258 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T10:42:38,258 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:42:38,259 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47726, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T10:42:38,259 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44ae20af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:42:38,260 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c91e417, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:42:38,260 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ea43b39fc24,35095,-1 for getting cluster id 2024-12-09T10:42:38,260 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T10:42:38,261 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0945b5ae-b46b-4b8f-af48-56b0bb92df09' 2024-12-09T10:42:38,261 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T10:42:38,261 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0945b5ae-b46b-4b8f-af48-56b0bb92df09" 2024-12-09T10:42:38,261 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@249b576c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:42:38,261 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ea43b39fc24,35095,-1] 2024-12-09T10:42:38,261 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T10:42:38,261 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:42:38,262 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47750, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T10:42:38,262 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40be6052, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:42:38,263 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a5d7882, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:42:38,263 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ea43b39fc24,35095,-1 for getting cluster id 2024-12-09T10:42:38,263 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T10:42:38,264 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0945b5ae-b46b-4b8f-af48-56b0bb92df09' 2024-12-09T10:42:38,264 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T10:42:38,264 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0945b5ae-b46b-4b8f-af48-56b0bb92df09" 2024-12-09T10:42:38,264 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f9b2ff4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:42:38,264 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ea43b39fc24,35095,-1] 2024-12-09T10:42:38,264 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T10:42:38,264 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:42:38,265 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47764, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T10:42:38,265 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7995dc20, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:42:38,266 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4907a519, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:42:38,266 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ea43b39fc24,35095,-1 for getting cluster id 2024-12-09T10:42:38,266 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T10:42:38,266 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0945b5ae-b46b-4b8f-af48-56b0bb92df09' 2024-12-09T10:42:38,266 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T10:42:38,266 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0945b5ae-b46b-4b8f-af48-56b0bb92df09" 2024-12-09T10:42:38,267 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35e349ce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:42:38,267 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ea43b39fc24,35095,-1] 2024-12-09T10:42:38,267 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T10:42:38,267 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:42:38,267 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47782, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T10:42:38,268 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e248e73, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:42:38,268 DEBUG [Thread-1930 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T10:42:38,269 DEBUG [Thread-1926 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T10:42:38,269 DEBUG [Thread-1932 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T10:42:38,269 DEBUG [Thread-1941 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T10:42:38,269 DEBUG [Thread-1928 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T10:42:38,269 DEBUG [Thread-1945 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T10:42:38,269 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ea43b39fc24,36101,1733740827678, seqNum=-1] 2024-12-09T10:42:38,269 DEBUG [Thread-1937 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T10:42:38,269 DEBUG [Thread-1939 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T10:42:38,270 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T10:42:38,270 DEBUG [Thread-1926 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ea43b39fc24,36101,1733740827678, seqNum=-1] 2024-12-09T10:42:38,270 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ea43b39fc24,36101,1733740827678, seqNum=-1] 2024-12-09T10:42:38,270 DEBUG [Thread-1926 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T10:42:38,270 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T10:42:38,270 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ea43b39fc24,36101,1733740827678, seqNum=-1] 2024-12-09T10:42:38,270 DEBUG [Thread-1943 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T10:42:38,270 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ea43b39fc24,36101,1733740827678, seqNum=-1] 2024-12-09T10:42:38,270 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T10:42:38,270 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T10:42:38,271 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ea43b39fc24,36101,1733740827678, seqNum=-1] 2024-12-09T10:42:38,271 DEBUG [Thread-1934 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T10:42:38,271 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T10:42:38,271 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59112, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T10:42:38,271 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59114, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T10:42:38,271 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59126, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T10:42:38,272 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59142, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T10:42:38,272 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-09T10:42:38,272 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ea43b39fc24,36101,1733740827678, seqNum=-1] 2024-12-09T10:42:38,272 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59136, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T10:42:38,272 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T10:42:38,272 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ea43b39fc24,36101,1733740827678, seqNum=-1] 2024-12-09T10:42:38,272 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ea43b39fc24,36101,1733740827678, seqNum=-1] 2024-12-09T10:42:38,272 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T10:42:38,272 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59138, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T10:42:38,273 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T10:42:38,273 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ea43b39fc24,36101,1733740827678, seqNum=-1] 2024-12-09T10:42:38,273 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T10:42:38,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2] 2024-12-09T10:42:38,273 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59150, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T10:42:38,274 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59156, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T10:42:38,274 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2] 2024-12-09T10:42:38,274 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2] 2024-12-09T10:42:38,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2] 2024-12-09T10:42:38,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-12-09T10:42:38,274 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59146, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T10:42:38,275 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59160, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T10:42:38,275 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2] 2024-12-09T10:42:38,276 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2] 2024-12-09T10:42:38,276 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T10:42:38,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-09T10:42:38,277 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2] 2024-12-09T10:42:38,277 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2] 2024-12-09T10:42:38,277 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T10:42:38,277 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2] 2024-12-09T10:42:38,277 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T10:42:38,278 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2] 2024-12-09T10:42:38,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(8855): Flush requested on a9d4962ae8c180b5b7fa587448872d04 2024-12-09T10:42:38,283 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a9d4962ae8c180b5b7fa587448872d04 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-09T10:42:38,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9d4962ae8c180b5b7fa587448872d04, store=A 2024-12-09T10:42:38,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:42:38,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9d4962ae8c180b5b7fa587448872d04, store=B 2024-12-09T10:42:38,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:42:38,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9d4962ae8c180b5b7fa587448872d04, store=C 2024-12-09T10:42:38,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:42:38,306 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/A/ede6cf7684cf4dc7ba6efe0144fb3879 is 50, key is test_row_0/A:col10/1733740958282/Put/seqid=0 2024-12-09T10:42:38,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742251_1427 (size=16681) 2024-12-09T10:42:38,361 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733740958328, max timestamp=1733740958357Num uniques 859; 2024-12-09T10:42:38,361 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733740958328, max timestamp=1733740958357Num uniques 859; 2024-12-09T10:42:38,361 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T10:42:38,361 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T10:42:38,361 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733740958328, max timestamp=1733740958357Num uniques 859; 2024-12-09T10:42:38,361 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T10:42:38,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-09T10:42:38,400 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:38,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59114 deadline: 1733741018389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:38,401 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:38,401 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:38,401 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:38,401 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:38,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59156 deadline: 1733741018390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:38,401 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:38,401 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:38,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59126 deadline: 1733741018390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:38,401 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:38,401 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:38,402 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:38,402 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:38,402 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:38,402 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:38,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59136 deadline: 1733741018391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:38,402 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:38,402 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:38,402 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:38,402 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:38,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59112 deadline: 1733741018401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:38,402 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:38,402 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:38,402 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:38,430 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-09T10:42:38,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. 2024-12-09T10:42:38,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. as already flushing 2024-12-09T10:42:38,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. 2024-12-09T10:42:38,431 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:38,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:38,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:38,519 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:38,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59114 deadline: 1733741018510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:38,519 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:38,519 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:38,519 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:38,519 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:38,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59156 deadline: 1733741018510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:38,520 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:38,520 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:38,520 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:38,520 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:38,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59126 deadline: 1733741018510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:38,520 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:38,520 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:38,520 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:38,520 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:38,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59136 deadline: 1733741018510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:38,520 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:38,520 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:38,520 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:38,520 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:38,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59112 deadline: 1733741018510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:38,520 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:38,520 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:38,521 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:38,583 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-09T10:42:38,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. 2024-12-09T10:42:38,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. as already flushing 2024-12-09T10:42:38,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. 2024-12-09T10:42:38,584 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:38,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:38,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:38,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-09T10:42:38,727 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/A/ede6cf7684cf4dc7ba6efe0144fb3879 2024-12-09T10:42:38,736 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-09T10:42:38,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. 2024-12-09T10:42:38,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. as already flushing 2024-12-09T10:42:38,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. 2024-12-09T10:42:38,737 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:38,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:38,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:38,740 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:38,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59114 deadline: 1733741018729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:38,740 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:38,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:38,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59156 deadline: 1733741018730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:38,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:38,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:38,740 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:38,740 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:38,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59126 deadline: 1733741018730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:38,740 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:38,740 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:38,741 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:38,741 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:38,741 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:38,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59136 deadline: 1733741018730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:38,741 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:38,741 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:38,741 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:38,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59112 deadline: 1733741018730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:38,741 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:38,741 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:38,741 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:38,741 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:38,741 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:38,749 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/B/91643f4180224083bde6bc6e1e0761e7 is 50, key is test_row_0/B:col10/1733740958282/Put/seqid=0 2024-12-09T10:42:38,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742252_1428 (size=12001) 2024-12-09T10:42:38,889 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-09T10:42:38,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. 2024-12-09T10:42:38,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. as already flushing 2024-12-09T10:42:38,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. 2024-12-09T10:42:38,890 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:38,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:38,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:38,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-09T10:42:39,042 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-09T10:42:39,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. 2024-12-09T10:42:39,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. as already flushing 2024-12-09T10:42:39,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. 2024-12-09T10:42:39,043 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:39,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:39,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:39,059 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:39,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59114 deadline: 1733741019049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:39,060 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:39,060 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:39,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59156 deadline: 1733741019050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:39,060 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:39,060 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:39,060 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:39,060 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:39,060 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:39,060 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:39,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59126 deadline: 1733741019050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:39,060 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:39,060 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:39,060 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:39,060 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:39,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59136 deadline: 1733741019050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:39,061 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:39,061 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:39,061 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:39,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59112 deadline: 1733741019050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:39,061 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:39,061 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:39,061 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:39,061 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:39,153 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/B/91643f4180224083bde6bc6e1e0761e7 2024-12-09T10:42:39,176 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/C/fc075148dd294d26a23184328dee613f is 50, key is test_row_0/C:col10/1733740958282/Put/seqid=0 2024-12-09T10:42:39,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742253_1429 (size=12001) 2024-12-09T10:42:39,196 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-09T10:42:39,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. 2024-12-09T10:42:39,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. as already flushing 2024-12-09T10:42:39,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. 2024-12-09T10:42:39,196 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:39,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:39,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:39,349 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-09T10:42:39,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. 2024-12-09T10:42:39,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. as already flushing 2024-12-09T10:42:39,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. 2024-12-09T10:42:39,349 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:39,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:39,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:39,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-09T10:42:39,502 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-09T10:42:39,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. 2024-12-09T10:42:39,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. as already flushing 2024-12-09T10:42:39,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. 2024-12-09T10:42:39,502 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:39,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:39,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:39,574 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:39,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59114 deadline: 1733741019569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:39,574 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:39,574 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:39,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:39,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59156 deadline: 1733741019569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:39,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:39,575 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:39,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59126 deadline: 1733741019569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:39,575 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:39,575 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:39,575 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:39,575 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:39,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59136 deadline: 1733741019569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:39,575 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:39,575 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:39,575 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:39,575 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:39,575 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:39,575 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:39,575 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:39,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59112 deadline: 1733741019569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:39,576 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:39,576 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:39,576 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:39,580 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/C/fc075148dd294d26a23184328dee613f 2024-12-09T10:42:39,584 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/A/ede6cf7684cf4dc7ba6efe0144fb3879 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/ede6cf7684cf4dc7ba6efe0144fb3879 2024-12-09T10:42:39,588 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/ede6cf7684cf4dc7ba6efe0144fb3879, entries=250, sequenceid=13, filesize=16.3 K 2024-12-09T10:42:39,588 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/B/91643f4180224083bde6bc6e1e0761e7 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/91643f4180224083bde6bc6e1e0761e7 2024-12-09T10:42:39,591 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/91643f4180224083bde6bc6e1e0761e7, entries=150, sequenceid=13, filesize=11.7 K 2024-12-09T10:42:39,592 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/C/fc075148dd294d26a23184328dee613f as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/fc075148dd294d26a23184328dee613f 2024-12-09T10:42:39,596 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/fc075148dd294d26a23184328dee613f, entries=150, sequenceid=13, filesize=11.7 K 2024-12-09T10:42:39,597 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=120.76 KB/123660 for a9d4962ae8c180b5b7fa587448872d04 in 1314ms, sequenceid=13, compaction requested=false 2024-12-09T10:42:39,597 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-09T10:42:39,598 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a9d4962ae8c180b5b7fa587448872d04: 2024-12-09T10:42:39,655 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-09T10:42:39,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. 2024-12-09T10:42:39,655 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing a9d4962ae8c180b5b7fa587448872d04 3/3 column families, dataSize=120.76 KB heapSize=340.45 KB 2024-12-09T10:42:39,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9d4962ae8c180b5b7fa587448872d04, store=A 2024-12-09T10:42:39,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T10:42:39,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9d4962ae8c180b5b7fa587448872d04, store=B 2024-12-09T10:42:39,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T10:42:39,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9d4962ae8c180b5b7fa587448872d04, store=C 2024-12-09T10:42:39,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T10:42:39,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/A/8885e4f83c1d48f2b3f5a3764b7624bb is 50, key is test_row_0/A:col10/1733740958399/Put/seqid=0 2024-12-09T10:42:39,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742254_1430 (size=12001) 2024-12-09T10:42:40,064 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/A/8885e4f83c1d48f2b3f5a3764b7624bb 2024-12-09T10:42:40,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/B/147c57e1cfce4d39a46151273fff679e is 50, key is test_row_0/B:col10/1733740958399/Put/seqid=0 2024-12-09T10:42:40,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742255_1431 (size=12001) 2024-12-09T10:42:40,075 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/B/147c57e1cfce4d39a46151273fff679e 2024-12-09T10:42:40,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/C/8c74baf9f4cf489da0805f3115b9de5f is 50, key is test_row_0/C:col10/1733740958399/Put/seqid=0 2024-12-09T10:42:40,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742256_1432 (size=12001) 2024-12-09T10:42:40,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-09T10:42:40,486 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/C/8c74baf9f4cf489da0805f3115b9de5f 2024-12-09T10:42:40,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/A/8885e4f83c1d48f2b3f5a3764b7624bb as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/8885e4f83c1d48f2b3f5a3764b7624bb 2024-12-09T10:42:40,493 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/8885e4f83c1d48f2b3f5a3764b7624bb, entries=150, sequenceid=47, filesize=11.7 K 2024-12-09T10:42:40,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/B/147c57e1cfce4d39a46151273fff679e as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/147c57e1cfce4d39a46151273fff679e 2024-12-09T10:42:40,497 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/147c57e1cfce4d39a46151273fff679e, entries=150, sequenceid=47, filesize=11.7 K 2024-12-09T10:42:40,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/C/8c74baf9f4cf489da0805f3115b9de5f as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/8c74baf9f4cf489da0805f3115b9de5f 2024-12-09T10:42:40,501 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/8c74baf9f4cf489da0805f3115b9de5f, entries=150, sequenceid=47, filesize=11.7 K 2024-12-09T10:42:40,510 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~340.41 KB/348576, currentSize=0 B/0 for a9d4962ae8c180b5b7fa587448872d04 in 854ms, sequenceid=47, compaction requested=false 2024-12-09T10:42:40,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for a9d4962ae8c180b5b7fa587448872d04: 2024-12-09T10:42:40,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. 2024-12-09T10:42:40,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-12-09T10:42:40,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-12-09T10:42:40,513 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-12-09T10:42:40,513 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2340 sec 2024-12-09T10:42:40,515 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 2.2420 sec 2024-12-09T10:42:40,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(8855): Flush requested on a9d4962ae8c180b5b7fa587448872d04 2024-12-09T10:42:40,596 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a9d4962ae8c180b5b7fa587448872d04 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-09T10:42:40,596 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9d4962ae8c180b5b7fa587448872d04, store=A 2024-12-09T10:42:40,596 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:42:40,596 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9d4962ae8c180b5b7fa587448872d04, store=B 2024-12-09T10:42:40,596 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:42:40,596 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9d4962ae8c180b5b7fa587448872d04, store=C 2024-12-09T10:42:40,596 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:42:40,601 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/A/a810fcf87a914544963f68a74ef348af is 50, key is test_row_0/A:col10/1733740960588/Put/seqid=0 2024-12-09T10:42:40,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742257_1433 (size=14341) 2024-12-09T10:42:40,634 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:40,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59114 deadline: 1733741020625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:40,634 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:40,635 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:40,635 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:40,635 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733740960617, max timestamp=1733740960632Num uniques 859; 2024-12-09T10:42:40,635 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733740960617, max timestamp=1733740960632Num uniques 859; 2024-12-09T10:42:40,635 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733740960617, max timestamp=1733740960632Num uniques 859; 2024-12-09T10:42:40,635 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T10:42:40,635 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T10:42:40,635 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T10:42:40,688 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:40,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59156 deadline: 1733741020671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:40,689 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:40,689 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:40,689 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:40,689 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:40,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59112 deadline: 1733741020683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:40,689 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:40,689 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:40,689 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:40,705 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:40,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59136 deadline: 1733741020687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:40,705 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:40,705 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:40,705 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:40,707 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:40,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59126 deadline: 1733741020689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:40,707 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:40,707 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:40,707 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:40,749 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:40,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59114 deadline: 1733741020739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:40,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:40,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:40,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:40,807 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:40,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59156 deadline: 1733741020800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:40,807 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:40,807 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:40,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59112 deadline: 1733741020800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:40,807 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:40,807 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:40,808 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:40,808 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:40,808 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:40,822 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:40,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59136 deadline: 1733741020809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:40,823 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:40,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59126 deadline: 1733741020809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:40,823 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:40,823 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:40,823 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:40,823 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:40,823 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:40,823 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:40,971 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:40,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59114 deadline: 1733741020959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:40,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:40,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:40,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:41,015 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=57 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/A/a810fcf87a914544963f68a74ef348af 2024-12-09T10:42:41,017 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:41,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59156 deadline: 1733741021009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:41,017 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:41,017 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:41,017 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:41,022 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/B/082a49cd746b411783e06a0445e2c82a is 50, key is test_row_0/B:col10/1733740960588/Put/seqid=0 2024-12-09T10:42:41,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742258_1434 (size=12001) 2024-12-09T10:42:41,032 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:41,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59112 deadline: 1733741021019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:41,032 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:41,033 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:41,033 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:41,041 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:41,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59136 deadline: 1733741021029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:41,042 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:41,042 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:41,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59126 deadline: 1733741021029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:41,042 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:41,042 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:41,042 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:41,042 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:41,042 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:41,169 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-09T10:42:41,290 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:41,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59114 deadline: 1733741021279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:41,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:41,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:41,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:41,329 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:41,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59156 deadline: 1733741021319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:41,330 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:41,330 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:41,330 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:41,353 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:41,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59112 deadline: 1733741021339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:41,353 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:41,354 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:41,354 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:41,362 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:41,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59136 deadline: 1733741021350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:41,362 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:41,363 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:41,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59126 deadline: 1733741021350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:41,363 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:41,363 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:41,363 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:41,363 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:41,363 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:41,426 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=57 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/B/082a49cd746b411783e06a0445e2c82a 2024-12-09T10:42:41,434 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/C/92a4e0df517f43ad90a32b359c891932 is 50, key is test_row_0/C:col10/1733740960588/Put/seqid=0 2024-12-09T10:42:41,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742259_1435 (size=12001) 2024-12-09T10:42:41,808 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:41,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59114 deadline: 1733741021799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:41,808 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:41,808 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:41,808 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:41,837 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=57 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/C/92a4e0df517f43ad90a32b359c891932 2024-12-09T10:42:41,842 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/A/a810fcf87a914544963f68a74ef348af as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/a810fcf87a914544963f68a74ef348af 2024-12-09T10:42:41,846 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/a810fcf87a914544963f68a74ef348af, entries=200, sequenceid=57, filesize=14.0 K 2024-12-09T10:42:41,846 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:41,847 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/B/082a49cd746b411783e06a0445e2c82a as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/082a49cd746b411783e06a0445e2c82a 2024-12-09T10:42:41,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59156 deadline: 1733741021840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:41,847 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:41,847 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:41,847 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:41,850 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/082a49cd746b411783e06a0445e2c82a, entries=150, sequenceid=57, filesize=11.7 K 2024-12-09T10:42:41,851 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/C/92a4e0df517f43ad90a32b359c891932 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/92a4e0df517f43ad90a32b359c891932 2024-12-09T10:42:41,855 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/92a4e0df517f43ad90a32b359c891932, entries=150, sequenceid=57, filesize=11.7 K 2024-12-09T10:42:41,855 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for a9d4962ae8c180b5b7fa587448872d04 in 1259ms, sequenceid=57, compaction requested=true 2024-12-09T10:42:41,855 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a9d4962ae8c180b5b7fa587448872d04: 2024-12-09T10:42:41,856 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9d4962ae8c180b5b7fa587448872d04:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T10:42:41,856 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T10:42:41,856 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:42:41,856 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9d4962ae8c180b5b7fa587448872d04:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T10:42:41,856 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:42:41,856 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9d4962ae8c180b5b7fa587448872d04:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T10:42:41,856 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T10:42:41,856 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T10:42:41,856 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T10:42:41,856 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43023 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T10:42:41,857 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1541): a9d4962ae8c180b5b7fa587448872d04/B is initiating minor compaction (all files) 2024-12-09T10:42:41,857 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1541): a9d4962ae8c180b5b7fa587448872d04/A is initiating minor compaction (all files) 2024-12-09T10:42:41,857 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a9d4962ae8c180b5b7fa587448872d04/B in TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. 2024-12-09T10:42:41,857 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a9d4962ae8c180b5b7fa587448872d04/A in TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. 2024-12-09T10:42:41,857 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/ede6cf7684cf4dc7ba6efe0144fb3879, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/8885e4f83c1d48f2b3f5a3764b7624bb, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/a810fcf87a914544963f68a74ef348af] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp, totalSize=42.0 K 2024-12-09T10:42:41,857 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/91643f4180224083bde6bc6e1e0761e7, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/147c57e1cfce4d39a46151273fff679e, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/082a49cd746b411783e06a0445e2c82a] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp, totalSize=35.2 K 2024-12-09T10:42:41,857 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting ede6cf7684cf4dc7ba6efe0144fb3879, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733740958278 2024-12-09T10:42:41,857 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 91643f4180224083bde6bc6e1e0761e7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733740958278 2024-12-09T10:42:41,857 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8885e4f83c1d48f2b3f5a3764b7624bb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733740958376 2024-12-09T10:42:41,857 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 147c57e1cfce4d39a46151273fff679e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733740958376 2024-12-09T10:42:41,858 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting a810fcf87a914544963f68a74ef348af, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1733740960586 2024-12-09T10:42:41,858 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 082a49cd746b411783e06a0445e2c82a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1733740960588 2024-12-09T10:42:41,864 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a9d4962ae8c180b5b7fa587448872d04#A#compaction#363 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:42:41,864 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/A/63b80a85f93a420497405757144f06e9 is 50, key is test_row_0/A:col10/1733740960588/Put/seqid=0 2024-12-09T10:42:41,867 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a9d4962ae8c180b5b7fa587448872d04#B#compaction#364 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:42:41,867 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/B/7c7bf0b746e941258bfb587778e6797e is 50, key is test_row_0/B:col10/1733740960588/Put/seqid=0 2024-12-09T10:42:41,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(8855): Flush requested on a9d4962ae8c180b5b7fa587448872d04 2024-12-09T10:42:41,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742260_1436 (size=12104) 2024-12-09T10:42:41,870 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a9d4962ae8c180b5b7fa587448872d04 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-12-09T10:42:41,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9d4962ae8c180b5b7fa587448872d04, store=A 2024-12-09T10:42:41,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T10:42:41,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9d4962ae8c180b5b7fa587448872d04, store=B 2024-12-09T10:42:41,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T10:42:41,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9d4962ae8c180b5b7fa587448872d04, store=C 2024-12-09T10:42:41,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T10:42:41,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742261_1437 (size=12104) 2024-12-09T10:42:41,875 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/A/0d982244b1bd4c1d8f8991e834f152e8 is 50, key is test_row_0/A:col10/1733740960682/Put/seqid=0 2024-12-09T10:42:41,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742262_1438 (size=14341) 2024-12-09T10:42:41,899 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:41,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59126 deadline: 1733741021883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:41,899 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:41,899 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:41,899 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:41,901 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:41,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59136 deadline: 1733741021889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:41,902 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:41,902 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:41,902 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:41,903 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:41,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59112 deadline: 1733741021899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:41,903 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:41,903 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:41,903 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:42,019 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:42,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59126 deadline: 1733741022010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:42,019 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:42,019 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:42,019 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:42,019 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:42,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59136 deadline: 1733741022010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:42,019 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:42,020 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:42,020 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:42,020 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:42,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59112 deadline: 1733741022010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:42,020 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:42,020 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:42,020 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:42,242 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:42,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59126 deadline: 1733741022230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:42,242 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:42,242 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:42,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59136 deadline: 1733741022230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:42,242 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:42,242 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:42,243 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:42,243 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:42,243 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:42,243 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:42,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59112 deadline: 1733741022230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:42,243 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:42,243 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:42,243 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:42,276 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/A/63b80a85f93a420497405757144f06e9 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/63b80a85f93a420497405757144f06e9 2024-12-09T10:42:42,278 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/A/0d982244b1bd4c1d8f8991e834f152e8 2024-12-09T10:42:42,279 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/B/7c7bf0b746e941258bfb587778e6797e as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/7c7bf0b746e941258bfb587778e6797e 2024-12-09T10:42:42,282 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a9d4962ae8c180b5b7fa587448872d04/A of a9d4962ae8c180b5b7fa587448872d04 into 63b80a85f93a420497405757144f06e9(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:42:42,282 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a9d4962ae8c180b5b7fa587448872d04: 2024-12-09T10:42:42,282 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., storeName=a9d4962ae8c180b5b7fa587448872d04/A, priority=13, startTime=1733740961855; duration=0sec 2024-12-09T10:42:42,282 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T10:42:42,282 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9d4962ae8c180b5b7fa587448872d04:A 2024-12-09T10:42:42,282 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T10:42:42,284 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T10:42:42,284 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1541): a9d4962ae8c180b5b7fa587448872d04/C is initiating minor compaction (all files) 2024-12-09T10:42:42,284 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a9d4962ae8c180b5b7fa587448872d04/C in TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. 2024-12-09T10:42:42,284 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/fc075148dd294d26a23184328dee613f, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/8c74baf9f4cf489da0805f3115b9de5f, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/92a4e0df517f43ad90a32b359c891932] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp, totalSize=35.2 K 2024-12-09T10:42:42,285 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting fc075148dd294d26a23184328dee613f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733740958278 2024-12-09T10:42:42,285 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8c74baf9f4cf489da0805f3115b9de5f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733740958376 2024-12-09T10:42:42,285 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 92a4e0df517f43ad90a32b359c891932, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1733740960588 2024-12-09T10:42:42,286 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a9d4962ae8c180b5b7fa587448872d04/B of a9d4962ae8c180b5b7fa587448872d04 into 7c7bf0b746e941258bfb587778e6797e(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:42:42,286 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a9d4962ae8c180b5b7fa587448872d04: 2024-12-09T10:42:42,286 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., storeName=a9d4962ae8c180b5b7fa587448872d04/B, priority=13, startTime=1733740961856; duration=0sec 2024-12-09T10:42:42,286 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:42:42,286 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9d4962ae8c180b5b7fa587448872d04:B 2024-12-09T10:42:42,287 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/B/5078744e03c24fcaa89b373b49cb40ed is 50, key is test_row_0/B:col10/1733740960682/Put/seqid=0 2024-12-09T10:42:42,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742263_1439 (size=12001) 2024-12-09T10:42:42,292 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a9d4962ae8c180b5b7fa587448872d04#C#compaction#367 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:42:42,292 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/B/5078744e03c24fcaa89b373b49cb40ed 2024-12-09T10:42:42,293 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/C/a4078266a2174e2ebab4f3417a0dc153 is 50, key is test_row_0/C:col10/1733740960588/Put/seqid=0 2024-12-09T10:42:42,300 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/C/e07da91f4679436ea54c2fa4102feb57 is 50, key is test_row_0/C:col10/1733740960682/Put/seqid=0 2024-12-09T10:42:42,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742264_1440 (size=12104) 2024-12-09T10:42:42,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742265_1441 (size=12001) 2024-12-09T10:42:42,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-09T10:42:42,449 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T10:42:42,450 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-09T10:42:42,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-12-09T10:42:42,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-09T10:42:42,452 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T10:42:42,452 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T10:42:42,452 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T10:42:42,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-09T10:42:42,561 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:42,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59126 deadline: 1733741022550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:42,561 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:42,561 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:42,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59136 deadline: 1733741022550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:42,561 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:42,561 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:42,562 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:42,562 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:42,562 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:42,562 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:42,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59112 deadline: 1733741022550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:42,562 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:42,562 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:42,562 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:42,604 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-09T10:42:42,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. 2024-12-09T10:42:42,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. as already flushing 2024-12-09T10:42:42,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. 2024-12-09T10:42:42,605 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:42,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:42,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:42,707 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/C/e07da91f4679436ea54c2fa4102feb57 2024-12-09T10:42:42,710 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/C/a4078266a2174e2ebab4f3417a0dc153 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/a4078266a2174e2ebab4f3417a0dc153 2024-12-09T10:42:42,711 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/A/0d982244b1bd4c1d8f8991e834f152e8 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/0d982244b1bd4c1d8f8991e834f152e8 2024-12-09T10:42:42,714 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/0d982244b1bd4c1d8f8991e834f152e8, entries=200, sequenceid=94, filesize=14.0 K 2024-12-09T10:42:42,715 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a9d4962ae8c180b5b7fa587448872d04/C of a9d4962ae8c180b5b7fa587448872d04 into a4078266a2174e2ebab4f3417a0dc153(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:42:42,715 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a9d4962ae8c180b5b7fa587448872d04: 2024-12-09T10:42:42,715 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., storeName=a9d4962ae8c180b5b7fa587448872d04/C, priority=13, startTime=1733740961856; duration=0sec 2024-12-09T10:42:42,715 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:42:42,715 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9d4962ae8c180b5b7fa587448872d04:C 2024-12-09T10:42:42,715 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/B/5078744e03c24fcaa89b373b49cb40ed as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/5078744e03c24fcaa89b373b49cb40ed 2024-12-09T10:42:42,718 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/5078744e03c24fcaa89b373b49cb40ed, entries=150, sequenceid=94, filesize=11.7 K 2024-12-09T10:42:42,719 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/C/e07da91f4679436ea54c2fa4102feb57 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/e07da91f4679436ea54c2fa4102feb57 2024-12-09T10:42:42,723 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/e07da91f4679436ea54c2fa4102feb57, entries=150, sequenceid=94, filesize=11.7 K 2024-12-09T10:42:42,723 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for a9d4962ae8c180b5b7fa587448872d04 in 854ms, sequenceid=94, compaction requested=false 2024-12-09T10:42:42,723 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a9d4962ae8c180b5b7fa587448872d04: 2024-12-09T10:42:42,757 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-09T10:42:42,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. 2024-12-09T10:42:42,758 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing a9d4962ae8c180b5b7fa587448872d04 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-12-09T10:42:42,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9d4962ae8c180b5b7fa587448872d04, store=A 2024-12-09T10:42:42,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:42:42,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9d4962ae8c180b5b7fa587448872d04, store=B 2024-12-09T10:42:42,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:42:42,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9d4962ae8c180b5b7fa587448872d04, store=C 2024-12-09T10:42:42,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:42:42,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/A/a77c95defe88421e9cc93b097e36a1e4 is 50, key is test_row_0/A:col10/1733740961887/Put/seqid=0 2024-12-09T10:42:42,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742266_1442 (size=9657) 2024-12-09T10:42:42,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-09T10:42:42,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(8855): Flush requested on a9d4962ae8c180b5b7fa587448872d04 2024-12-09T10:42:42,823 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. as already flushing 2024-12-09T10:42:42,908 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733740962879, max timestamp=1733740962904Num uniques 859; 2024-12-09T10:42:42,908 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-09T10:42:42,908 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733740962879, max timestamp=1733740962904Num uniques 859; 2024-12-09T10:42:42,908 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-09T10:42:42,908 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733740962879, max timestamp=1733740962904Num uniques 859; 2024-12-09T10:42:42,908 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-09T10:42:43,006 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:43,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59156 deadline: 1733741022996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:43,006 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:43,006 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:43,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59114 deadline: 1733741022996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:43,006 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:43,006 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:43,006 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:43,006 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:43,006 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:43,073 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:43,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59126 deadline: 1733741023070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:43,074 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:43,074 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:43,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59136 deadline: 1733741023070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:43,074 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:43,074 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:43,074 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:43,074 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:43,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59112 deadline: 1733741023070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:43,074 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:43,074 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:43,074 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:43,074 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:43,074 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:43,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-09T10:42:43,113 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:43,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59156 deadline: 1733741023110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:43,113 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:43,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59114 deadline: 1733741023110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:43,113 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:43,113 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:43,113 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:43,114 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:43,114 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:43,114 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:43,166 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=106 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/A/a77c95defe88421e9cc93b097e36a1e4 2024-12-09T10:42:43,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/B/824f3705cc1f49528a8254d1439c06b0 is 50, key is test_row_0/B:col10/1733740961887/Put/seqid=0 2024-12-09T10:42:43,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742267_1443 (size=9657) 2024-12-09T10:42:43,328 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:43,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59156 deadline: 1733741023319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:43,328 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:43,328 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:43,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59114 deadline: 1733741023319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:43,328 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:43,328 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:43,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:43,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:43,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:43,582 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=106 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/B/824f3705cc1f49528a8254d1439c06b0 2024-12-09T10:42:43,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/C/3781b717cc594df3958a69e047ec4176 is 50, key is test_row_0/C:col10/1733740961887/Put/seqid=0 2024-12-09T10:42:43,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-09T10:42:43,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742268_1444 (size=9657) 2024-12-09T10:42:43,648 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:43,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59156 deadline: 1733741023639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:43,648 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:43,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59114 deadline: 1733741023639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:43,649 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:43,649 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:43,649 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:43,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:43,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:43,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:43,992 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=106 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/C/3781b717cc594df3958a69e047ec4176 2024-12-09T10:42:43,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/A/a77c95defe88421e9cc93b097e36a1e4 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/a77c95defe88421e9cc93b097e36a1e4 2024-12-09T10:42:44,000 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/a77c95defe88421e9cc93b097e36a1e4, entries=100, sequenceid=106, filesize=9.4 K 2024-12-09T10:42:44,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/B/824f3705cc1f49528a8254d1439c06b0 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/824f3705cc1f49528a8254d1439c06b0 2024-12-09T10:42:44,005 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/824f3705cc1f49528a8254d1439c06b0, entries=100, sequenceid=106, filesize=9.4 K 2024-12-09T10:42:44,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/C/3781b717cc594df3958a69e047ec4176 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/3781b717cc594df3958a69e047ec4176 2024-12-09T10:42:44,011 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/3781b717cc594df3958a69e047ec4176, entries=100, sequenceid=106, filesize=9.4 K 2024-12-09T10:42:44,012 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=147.60 KB/151140 for a9d4962ae8c180b5b7fa587448872d04 in 1255ms, sequenceid=106, compaction requested=true 2024-12-09T10:42:44,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for a9d4962ae8c180b5b7fa587448872d04: 2024-12-09T10:42:44,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. 2024-12-09T10:42:44,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-12-09T10:42:44,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-12-09T10:42:44,015 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-12-09T10:42:44,016 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5620 sec 2024-12-09T10:42:44,017 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 1.5660 sec 2024-12-09T10:42:44,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(8855): Flush requested on a9d4962ae8c180b5b7fa587448872d04 2024-12-09T10:42:44,092 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a9d4962ae8c180b5b7fa587448872d04 3/3 column families, dataSize=154.31 KB heapSize=436.08 KB 2024-12-09T10:42:44,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9d4962ae8c180b5b7fa587448872d04, store=A 2024-12-09T10:42:44,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-09T10:42:44,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9d4962ae8c180b5b7fa587448872d04, store=B 2024-12-09T10:42:44,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-09T10:42:44,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9d4962ae8c180b5b7fa587448872d04, store=C 2024-12-09T10:42:44,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-09T10:42:44,101 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/A/e78d092557904c17beda2f7b0aea08b6 is 50, key is test_row_0/A:col10/1733740964092/Put/seqid=0 2024-12-09T10:42:44,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742269_1445 (size=14541) 2024-12-09T10:42:44,108 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:44,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59112 deadline: 1733741024097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:44,109 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:44,109 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:44,109 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:44,112 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:44,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59126 deadline: 1733741024099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:44,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:44,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:44,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:44,125 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:44,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59136 deadline: 1733741024108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:44,126 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:44,126 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:44,126 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:44,167 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:44,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59156 deadline: 1733741024160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:44,168 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:44,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59114 deadline: 1733741024160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:44,168 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:44,168 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:44,168 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:44,168 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:44,168 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:44,168 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:44,225 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:44,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59112 deadline: 1733741024219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:44,225 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:44,225 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:44,226 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:44,231 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:44,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59126 deadline: 1733741024220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:44,232 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:44,232 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:44,232 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:44,239 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:44,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59136 deadline: 1733741024230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:44,240 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:44,240 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:44,240 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:44,438 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:44,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59112 deadline: 1733741024430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:44,439 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:44,439 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:44,439 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:44,451 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:44,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59126 deadline: 1733741024440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:44,452 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:44,452 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:44,452 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:44,459 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:44,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59136 deadline: 1733741024449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:44,460 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:44,460 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:44,460 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:44,505 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/A/e78d092557904c17beda2f7b0aea08b6 2024-12-09T10:42:44,513 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/B/815471661efe40d29a8b77b7552b6c3c is 50, key is test_row_0/B:col10/1733740964092/Put/seqid=0 2024-12-09T10:42:44,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742270_1446 (size=12151) 2024-12-09T10:42:44,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-09T10:42:44,600 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T10:42:44,600 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-09T10:42:44,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-12-09T10:42:44,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-09T10:42:44,602 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T10:42:44,603 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T10:42:44,603 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T10:42:44,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-09T10:42:44,755 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-09T10:42:44,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. 2024-12-09T10:42:44,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. as already flushing 2024-12-09T10:42:44,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. 2024-12-09T10:42:44,755 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:44,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:44,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:44,761 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:44,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59112 deadline: 1733741024750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:44,762 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:44,762 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:44,762 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:44,764 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:44,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59126 deadline: 1733741024759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:44,765 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:44,765 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:44,765 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:44,778 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:44,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59136 deadline: 1733741024769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:44,779 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:44,779 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:44,779 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:44,908 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-09T10:42:44,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. 2024-12-09T10:42:44,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. as already flushing 2024-12-09T10:42:44,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. 2024-12-09T10:42:44,909 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:44,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:44,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:44,920 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/B/815471661efe40d29a8b77b7552b6c3c 2024-12-09T10:42:44,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-09T10:42:44,940 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/C/2d2fc7f4b82b425e9af977c87f18319d is 50, key is test_row_0/C:col10/1733740964092/Put/seqid=0 2024-12-09T10:42:44,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742271_1447 (size=12151) 2024-12-09T10:42:44,953 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/C/2d2fc7f4b82b425e9af977c87f18319d 2024-12-09T10:42:44,960 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/A/e78d092557904c17beda2f7b0aea08b6 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/e78d092557904c17beda2f7b0aea08b6 2024-12-09T10:42:44,964 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/e78d092557904c17beda2f7b0aea08b6, entries=200, sequenceid=145, filesize=14.2 K 2024-12-09T10:42:44,972 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/B/815471661efe40d29a8b77b7552b6c3c as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/815471661efe40d29a8b77b7552b6c3c 2024-12-09T10:42:44,978 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/815471661efe40d29a8b77b7552b6c3c, entries=150, sequenceid=145, filesize=11.9 K 2024-12-09T10:42:44,979 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/C/2d2fc7f4b82b425e9af977c87f18319d as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/2d2fc7f4b82b425e9af977c87f18319d 2024-12-09T10:42:44,984 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/2d2fc7f4b82b425e9af977c87f18319d, entries=150, sequenceid=145, filesize=11.9 K 2024-12-09T10:42:44,985 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~455.95 KB/466896, currentSize=20.13 KB/20610 for a9d4962ae8c180b5b7fa587448872d04 in 892ms, sequenceid=145, compaction requested=true 2024-12-09T10:42:44,985 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a9d4962ae8c180b5b7fa587448872d04: 2024-12-09T10:42:44,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9d4962ae8c180b5b7fa587448872d04:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T10:42:44,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:42:44,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9d4962ae8c180b5b7fa587448872d04:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T10:42:44,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:42:44,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9d4962ae8c180b5b7fa587448872d04:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T10:42:44,985 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T10:42:44,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T10:42:44,985 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T10:42:44,986 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50643 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T10:42:44,987 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1541): a9d4962ae8c180b5b7fa587448872d04/A is initiating minor compaction (all files) 2024-12-09T10:42:44,987 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45913 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T10:42:44,987 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1541): a9d4962ae8c180b5b7fa587448872d04/B is initiating minor compaction (all files) 2024-12-09T10:42:44,987 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a9d4962ae8c180b5b7fa587448872d04/A in TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. 2024-12-09T10:42:44,987 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a9d4962ae8c180b5b7fa587448872d04/B in TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. 2024-12-09T10:42:44,987 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/7c7bf0b746e941258bfb587778e6797e, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/5078744e03c24fcaa89b373b49cb40ed, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/824f3705cc1f49528a8254d1439c06b0, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/815471661efe40d29a8b77b7552b6c3c] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp, totalSize=44.8 K 2024-12-09T10:42:44,987 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/63b80a85f93a420497405757144f06e9, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/0d982244b1bd4c1d8f8991e834f152e8, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/a77c95defe88421e9cc93b097e36a1e4, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/e78d092557904c17beda2f7b0aea08b6] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp, totalSize=49.5 K 2024-12-09T10:42:44,987 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 63b80a85f93a420497405757144f06e9, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1733740960588 2024-12-09T10:42:44,987 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c7bf0b746e941258bfb587778e6797e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1733740960588 2024-12-09T10:42:44,988 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0d982244b1bd4c1d8f8991e834f152e8, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1733740960682 2024-12-09T10:42:44,988 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 5078744e03c24fcaa89b373b49cb40ed, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1733740960682 2024-12-09T10:42:44,988 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting a77c95defe88421e9cc93b097e36a1e4, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=106, earliestPutTs=1733740961887 2024-12-09T10:42:44,988 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 824f3705cc1f49528a8254d1439c06b0, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=106, earliestPutTs=1733740961887 2024-12-09T10:42:44,988 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 815471661efe40d29a8b77b7552b6c3c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1733740962994 2024-12-09T10:42:44,988 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting e78d092557904c17beda2f7b0aea08b6, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1733740962994 2024-12-09T10:42:45,008 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a9d4962ae8c180b5b7fa587448872d04#B#compaction#375 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:42:45,010 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a9d4962ae8c180b5b7fa587448872d04#A#compaction#376 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:42:45,010 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/A/3fe0357bead24995ab0df9001b0a758a is 50, key is test_row_0/A:col10/1733740964092/Put/seqid=0 2024-12-09T10:42:45,013 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/B/7621085d7d964420a2c0f71470975b91 is 50, key is test_row_0/B:col10/1733740964092/Put/seqid=0 2024-12-09T10:42:45,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742272_1448 (size=12391) 2024-12-09T10:42:45,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742273_1449 (size=12391) 2024-12-09T10:42:45,062 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-09T10:42:45,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. 2024-12-09T10:42:45,062 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing a9d4962ae8c180b5b7fa587448872d04 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-12-09T10:42:45,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9d4962ae8c180b5b7fa587448872d04, store=A 2024-12-09T10:42:45,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:42:45,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9d4962ae8c180b5b7fa587448872d04, store=B 2024-12-09T10:42:45,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:42:45,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9d4962ae8c180b5b7fa587448872d04, store=C 2024-12-09T10:42:45,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:42:45,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/A/6eebef8d6f96424db6d866097fd1c8c6 is 50, key is test_row_1/A:col10/1733740964095/Put/seqid=0 2024-12-09T10:42:45,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742274_1450 (size=9757) 2024-12-09T10:42:45,076 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/A/6eebef8d6f96424db6d866097fd1c8c6 2024-12-09T10:42:45,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/B/27fdc4df16ac436cb9d58bef806c5b09 is 50, key is test_row_1/B:col10/1733740964095/Put/seqid=0 2024-12-09T10:42:45,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742275_1451 (size=9757) 2024-12-09T10:42:45,095 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/B/27fdc4df16ac436cb9d58bef806c5b09 2024-12-09T10:42:45,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/C/7021dd8209f246229cad52ca59f31fe9 is 50, key is test_row_1/C:col10/1733740964095/Put/seqid=0 2024-12-09T10:42:45,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742276_1452 (size=9757) 2024-12-09T10:42:45,107 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/C/7021dd8209f246229cad52ca59f31fe9 2024-12-09T10:42:45,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/A/6eebef8d6f96424db6d866097fd1c8c6 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/6eebef8d6f96424db6d866097fd1c8c6 2024-12-09T10:42:45,114 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/6eebef8d6f96424db6d866097fd1c8c6, entries=100, sequenceid=151, filesize=9.5 K 2024-12-09T10:42:45,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/B/27fdc4df16ac436cb9d58bef806c5b09 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/27fdc4df16ac436cb9d58bef806c5b09 2024-12-09T10:42:45,119 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/27fdc4df16ac436cb9d58bef806c5b09, entries=100, sequenceid=151, filesize=9.5 K 2024-12-09T10:42:45,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/C/7021dd8209f246229cad52ca59f31fe9 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/7021dd8209f246229cad52ca59f31fe9 2024-12-09T10:42:45,123 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/7021dd8209f246229cad52ca59f31fe9, entries=100, sequenceid=151, filesize=9.5 K 2024-12-09T10:42:45,124 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for a9d4962ae8c180b5b7fa587448872d04 in 62ms, sequenceid=151, compaction requested=true 2024-12-09T10:42:45,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for a9d4962ae8c180b5b7fa587448872d04: 2024-12-09T10:42:45,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. 2024-12-09T10:42:45,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-12-09T10:42:45,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-12-09T10:42:45,130 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-12-09T10:42:45,130 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 523 msec 2024-12-09T10:42:45,132 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 530 msec 2024-12-09T10:42:45,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(8855): Flush requested on a9d4962ae8c180b5b7fa587448872d04 2024-12-09T10:42:45,194 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a9d4962ae8c180b5b7fa587448872d04 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-09T10:42:45,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9d4962ae8c180b5b7fa587448872d04, store=A 2024-12-09T10:42:45,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:42:45,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9d4962ae8c180b5b7fa587448872d04, store=B 2024-12-09T10:42:45,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:42:45,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9d4962ae8c180b5b7fa587448872d04, store=C 2024-12-09T10:42:45,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:42:45,198 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/A/d6986f3396e84565b5cc5755f71cda4c is 50, key is test_row_0/A:col10/1733740965193/Put/seqid=0 2024-12-09T10:42:45,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742277_1453 (size=14537) 2024-12-09T10:42:45,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-09T10:42:45,230 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T10:42:45,231 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-09T10:42:45,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-12-09T10:42:45,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-09T10:42:45,232 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T10:42:45,233 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T10:42:45,233 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T10:42:45,274 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733740965237, max timestamp=1733740965271Num uniques 859; 2024-12-09T10:42:45,274 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T10:42:45,274 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733740965237, max timestamp=1733740965271Num uniques 859; 2024-12-09T10:42:45,274 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T10:42:45,275 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733740965237, max timestamp=1733740965271Num uniques 859; 2024-12-09T10:42:45,275 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T10:42:45,321 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:45,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59112 deadline: 1733741025311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:45,322 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:45,322 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:45,322 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:45,328 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:45,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59126 deadline: 1733741025312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:45,328 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:45,328 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:45,328 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:45,334 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:45,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59156 deadline: 1733741025317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:45,334 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:45,334 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:45,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59136 deadline: 1733741025318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:45,335 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:45,335 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:45,335 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:45,335 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:45,335 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:45,335 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:45,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59114 deadline: 1733741025319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:45,335 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:45,335 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:45,335 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:45,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-09T10:42:45,385 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-09T10:42:45,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. 2024-12-09T10:42:45,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. as already flushing 2024-12-09T10:42:45,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. 2024-12-09T10:42:45,386 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:45,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:45,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:45,436 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:45,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59112 deadline: 1733741025429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:45,437 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:45,437 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:45,437 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:45,440 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/A/3fe0357bead24995ab0df9001b0a758a as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/3fe0357bead24995ab0df9001b0a758a 2024-12-09T10:42:45,440 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/B/7621085d7d964420a2c0f71470975b91 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/7621085d7d964420a2c0f71470975b91 2024-12-09T10:42:45,445 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a9d4962ae8c180b5b7fa587448872d04/A of a9d4962ae8c180b5b7fa587448872d04 into 3fe0357bead24995ab0df9001b0a758a(size=12.1 K), total size for store is 21.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:42:45,445 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a9d4962ae8c180b5b7fa587448872d04/B of a9d4962ae8c180b5b7fa587448872d04 into 7621085d7d964420a2c0f71470975b91(size=12.1 K), total size for store is 21.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:42:45,445 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a9d4962ae8c180b5b7fa587448872d04: 2024-12-09T10:42:45,445 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a9d4962ae8c180b5b7fa587448872d04: 2024-12-09T10:42:45,445 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., storeName=a9d4962ae8c180b5b7fa587448872d04/A, priority=12, startTime=1733740964985; duration=0sec 2024-12-09T10:42:45,445 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., storeName=a9d4962ae8c180b5b7fa587448872d04/B, priority=12, startTime=1733740964985; duration=0sec 2024-12-09T10:42:45,445 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T10:42:45,445 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9d4962ae8c180b5b7fa587448872d04:A 2024-12-09T10:42:45,445 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T10:42:45,445 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9d4962ae8c180b5b7fa587448872d04:B 2024-12-09T10:42:45,445 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-09T10:42:45,446 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 55670 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-09T10:42:45,446 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1541): a9d4962ae8c180b5b7fa587448872d04/C is initiating minor compaction (all files) 2024-12-09T10:42:45,447 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a9d4962ae8c180b5b7fa587448872d04/C in TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. 2024-12-09T10:42:45,447 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/a4078266a2174e2ebab4f3417a0dc153, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/e07da91f4679436ea54c2fa4102feb57, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/3781b717cc594df3958a69e047ec4176, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/2d2fc7f4b82b425e9af977c87f18319d, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/7021dd8209f246229cad52ca59f31fe9] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp, totalSize=54.4 K 2024-12-09T10:42:45,447 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting a4078266a2174e2ebab4f3417a0dc153, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1733740960588 2024-12-09T10:42:45,447 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting e07da91f4679436ea54c2fa4102feb57, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1733740960682 2024-12-09T10:42:45,448 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3781b717cc594df3958a69e047ec4176, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=106, earliestPutTs=1733740961887 2024-12-09T10:42:45,448 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2d2fc7f4b82b425e9af977c87f18319d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1733740962994 2024-12-09T10:42:45,448 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7021dd8209f246229cad52ca59f31fe9, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1733740964095 2024-12-09T10:42:45,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:45,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59136 deadline: 1733741025440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:45,454 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:45,454 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:45,454 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:45,454 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:45,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59126 deadline: 1733741025440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:45,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:45,454 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:45,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:45,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:45,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59114 deadline: 1733741025440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:45,455 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:45,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:45,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59156 deadline: 1733741025440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:45,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:45,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:45,455 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:45,455 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:45,455 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:45,456 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a9d4962ae8c180b5b7fa587448872d04#C#compaction#381 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:42:45,456 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/C/c1c5efacdb2f46b9bfec1e38e4c0fc6e is 50, key is test_row_0/C:col10/1733740964092/Put/seqid=0 2024-12-09T10:42:45,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742278_1454 (size=12425) 2024-12-09T10:42:45,539 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-09T10:42:45,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. 2024-12-09T10:42:45,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. as already flushing 2024-12-09T10:42:45,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. 2024-12-09T10:42:45,539 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:45,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:45,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:45,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-09T10:42:45,602 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=161 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/A/d6986f3396e84565b5cc5755f71cda4c 2024-12-09T10:42:45,609 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/B/d73bb15157a146258389d3e6e09218bd is 50, key is test_row_0/B:col10/1733740965193/Put/seqid=0 2024-12-09T10:42:45,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742279_1455 (size=9757) 2024-12-09T10:42:45,652 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:45,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59112 deadline: 1733741025640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:45,653 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:45,653 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:45,653 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:45,667 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:45,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59136 deadline: 1733741025659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:45,668 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:45,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59126 deadline: 1733741025659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:45,668 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:45,668 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:45,668 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:45,668 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:45,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59156 deadline: 1733741025659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:45,668 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:45,668 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:45,668 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:45,668 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:45,668 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:45,668 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:45,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59114 deadline: 1733741025660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:45,668 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:45,668 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:45,668 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:45,668 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:45,692 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-09T10:42:45,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. 2024-12-09T10:42:45,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. as already flushing 2024-12-09T10:42:45,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. 2024-12-09T10:42:45,693 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:45,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:45,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:45,845 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-09T10:42:45,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. 2024-12-09T10:42:45,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. as already flushing 2024-12-09T10:42:45,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. 2024-12-09T10:42:45,846 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:45,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:45,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:45,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-09T10:42:45,863 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/C/c1c5efacdb2f46b9bfec1e38e4c0fc6e as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/c1c5efacdb2f46b9bfec1e38e4c0fc6e 2024-12-09T10:42:45,868 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in a9d4962ae8c180b5b7fa587448872d04/C of a9d4962ae8c180b5b7fa587448872d04 into c1c5efacdb2f46b9bfec1e38e4c0fc6e(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:42:45,868 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a9d4962ae8c180b5b7fa587448872d04: 2024-12-09T10:42:45,868 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., storeName=a9d4962ae8c180b5b7fa587448872d04/C, priority=11, startTime=1733740964985; duration=0sec 2024-12-09T10:42:45,868 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:42:45,868 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9d4962ae8c180b5b7fa587448872d04:C 2024-12-09T10:42:45,968 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:45,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59112 deadline: 1733741025959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:45,969 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:45,969 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:45,969 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:45,976 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:45,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59136 deadline: 1733741025969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:45,977 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:45,977 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:45,977 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:45,991 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:45,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59126 deadline: 1733741025980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:45,991 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:45,991 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:45,991 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:45,991 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:45,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59156 deadline: 1733741025980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:45,991 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:45,991 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:45,991 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:45,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59114 deadline: 1733741025980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:45,991 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:45,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:45,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:45,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:45,998 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-09T10:42:45,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. 2024-12-09T10:42:45,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. as already flushing 2024-12-09T10:42:45,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. 2024-12-09T10:42:45,999 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:45,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:46,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:46,013 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=161 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/B/d73bb15157a146258389d3e6e09218bd 2024-12-09T10:42:46,019 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/C/fe66cb496d904e0da0d60997990f353b is 50, key is test_row_0/C:col10/1733740965193/Put/seqid=0 2024-12-09T10:42:46,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742280_1456 (size=9757) 2024-12-09T10:42:46,151 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-09T10:42:46,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. 2024-12-09T10:42:46,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. as already flushing 2024-12-09T10:42:46,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. 2024-12-09T10:42:46,152 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:46,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:46,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:46,305 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-09T10:42:46,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. 2024-12-09T10:42:46,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. as already flushing 2024-12-09T10:42:46,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. 2024-12-09T10:42:46,305 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:46,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:46,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:46,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-09T10:42:46,423 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=161 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/C/fe66cb496d904e0da0d60997990f353b 2024-12-09T10:42:46,428 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/A/d6986f3396e84565b5cc5755f71cda4c as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/d6986f3396e84565b5cc5755f71cda4c 2024-12-09T10:42:46,431 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/d6986f3396e84565b5cc5755f71cda4c, entries=200, sequenceid=161, filesize=14.2 K 2024-12-09T10:42:46,432 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/B/d73bb15157a146258389d3e6e09218bd as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/d73bb15157a146258389d3e6e09218bd 2024-12-09T10:42:46,435 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/d73bb15157a146258389d3e6e09218bd, entries=100, sequenceid=161, filesize=9.5 K 2024-12-09T10:42:46,436 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/C/fe66cb496d904e0da0d60997990f353b as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/fe66cb496d904e0da0d60997990f353b 2024-12-09T10:42:46,440 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/fe66cb496d904e0da0d60997990f353b, entries=100, sequenceid=161, filesize=9.5 K 2024-12-09T10:42:46,441 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for a9d4962ae8c180b5b7fa587448872d04 in 1247ms, sequenceid=161, compaction requested=true 2024-12-09T10:42:46,441 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a9d4962ae8c180b5b7fa587448872d04: 2024-12-09T10:42:46,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9d4962ae8c180b5b7fa587448872d04:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T10:42:46,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:42:46,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9d4962ae8c180b5b7fa587448872d04:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T10:42:46,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:42:46,441 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T10:42:46,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9d4962ae8c180b5b7fa587448872d04:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T10:42:46,441 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T10:42:46,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T10:42:46,442 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 31905 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T10:42:46,442 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1541): a9d4962ae8c180b5b7fa587448872d04/B is initiating minor compaction (all files) 2024-12-09T10:42:46,442 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a9d4962ae8c180b5b7fa587448872d04/B in TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. 2024-12-09T10:42:46,442 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/7621085d7d964420a2c0f71470975b91, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/27fdc4df16ac436cb9d58bef806c5b09, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/d73bb15157a146258389d3e6e09218bd] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp, totalSize=31.2 K 2024-12-09T10:42:46,442 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36685 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T10:42:46,442 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1541): a9d4962ae8c180b5b7fa587448872d04/A is initiating minor compaction (all files) 2024-12-09T10:42:46,442 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a9d4962ae8c180b5b7fa587448872d04/A in TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. 2024-12-09T10:42:46,442 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/3fe0357bead24995ab0df9001b0a758a, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/6eebef8d6f96424db6d866097fd1c8c6, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/d6986f3396e84565b5cc5755f71cda4c] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp, totalSize=35.8 K 2024-12-09T10:42:46,442 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 7621085d7d964420a2c0f71470975b91, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1733740962994 2024-12-09T10:42:46,443 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3fe0357bead24995ab0df9001b0a758a, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1733740962994 2024-12-09T10:42:46,443 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 27fdc4df16ac436cb9d58bef806c5b09, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1733740964095 2024-12-09T10:42:46,443 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6eebef8d6f96424db6d866097fd1c8c6, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1733740964095 2024-12-09T10:42:46,443 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting d73bb15157a146258389d3e6e09218bd, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1733740965192 2024-12-09T10:42:46,443 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting d6986f3396e84565b5cc5755f71cda4c, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1733740965182 2024-12-09T10:42:46,449 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a9d4962ae8c180b5b7fa587448872d04#A#compaction#384 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:42:46,450 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/A/9bb012ad13804bf7a0596b186ed951af is 50, key is test_row_0/A:col10/1733740965193/Put/seqid=0 2024-12-09T10:42:46,452 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a9d4962ae8c180b5b7fa587448872d04#B#compaction#385 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:42:46,452 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/B/0590e692e7154225a6985b40d5133ae8 is 50, key is test_row_0/B:col10/1733740965193/Put/seqid=0 2024-12-09T10:42:46,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742282_1458 (size=12493) 2024-12-09T10:42:46,459 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-09T10:42:46,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. 2024-12-09T10:42:46,459 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing a9d4962ae8c180b5b7fa587448872d04 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-12-09T10:42:46,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9d4962ae8c180b5b7fa587448872d04, store=A 2024-12-09T10:42:46,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T10:42:46,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9d4962ae8c180b5b7fa587448872d04, store=B 2024-12-09T10:42:46,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T10:42:46,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9d4962ae8c180b5b7fa587448872d04, store=C 2024-12-09T10:42:46,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T10:42:46,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/A/58304220a6644c908c5a8c77acbe05d8 is 50, key is test_row_0/A:col10/1733740965316/Put/seqid=0 2024-12-09T10:42:46,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742281_1457 (size=12493) 2024-12-09T10:42:46,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742283_1459 (size=12151) 2024-12-09T10:42:46,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(8855): Flush requested on a9d4962ae8c180b5b7fa587448872d04 2024-12-09T10:42:46,488 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. as already flushing 2024-12-09T10:42:46,509 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:46,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59114 deadline: 1733741026500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:46,509 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:46,510 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:46,510 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:46,510 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:46,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59112 deadline: 1733741026500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:46,510 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:46,510 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:46,510 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:46,513 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:46,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59136 deadline: 1733741026507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:46,514 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:46,514 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:46,514 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:46,524 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:46,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59126 deadline: 1733741026508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:46,524 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:46,524 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:46,524 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:46,524 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:46,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59156 deadline: 1733741026509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:46,524 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:46,524 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:46,524 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:46,632 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:46,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59112 deadline: 1733741026620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:46,632 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:46,632 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:46,632 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:46,632 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:46,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59136 deadline: 1733741026620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:46,632 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:46,632 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:46,632 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:46,642 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:46,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59126 deadline: 1733741026629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:46,642 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:46,642 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:46,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59156 deadline: 1733741026629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:46,642 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:46,642 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:46,642 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:46,642 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:46,642 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:46,852 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:46,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59112 deadline: 1733741026840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:46,852 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:46,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59136 deadline: 1733741026840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:46,852 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:46,852 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:46,852 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:46,853 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:46,853 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:46,853 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:46,857 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:46,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59126 deadline: 1733741026849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:46,858 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:46,858 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:46,858 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:46,858 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:46,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59156 deadline: 1733741026849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:46,858 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:46,858 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:46,858 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:46,864 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/B/0590e692e7154225a6985b40d5133ae8 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/0590e692e7154225a6985b40d5133ae8 2024-12-09T10:42:46,868 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a9d4962ae8c180b5b7fa587448872d04/B of a9d4962ae8c180b5b7fa587448872d04 into 0590e692e7154225a6985b40d5133ae8(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:42:46,868 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a9d4962ae8c180b5b7fa587448872d04: 2024-12-09T10:42:46,868 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., storeName=a9d4962ae8c180b5b7fa587448872d04/B, priority=13, startTime=1733740966441; duration=0sec 2024-12-09T10:42:46,868 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T10:42:46,868 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9d4962ae8c180b5b7fa587448872d04:B 2024-12-09T10:42:46,868 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/A/9bb012ad13804bf7a0596b186ed951af as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/9bb012ad13804bf7a0596b186ed951af 2024-12-09T10:42:46,868 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-09T10:42:46,870 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-09T10:42:46,870 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-09T10:42:46,870 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. because compaction request was cancelled 2024-12-09T10:42:46,870 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9d4962ae8c180b5b7fa587448872d04:C 2024-12-09T10:42:46,871 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/A/58304220a6644c908c5a8c77acbe05d8 2024-12-09T10:42:46,874 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a9d4962ae8c180b5b7fa587448872d04/A of a9d4962ae8c180b5b7fa587448872d04 into 9bb012ad13804bf7a0596b186ed951af(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:42:46,874 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a9d4962ae8c180b5b7fa587448872d04: 2024-12-09T10:42:46,874 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., storeName=a9d4962ae8c180b5b7fa587448872d04/A, priority=13, startTime=1733740966441; duration=0sec 2024-12-09T10:42:46,874 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:42:46,874 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9d4962ae8c180b5b7fa587448872d04:A 2024-12-09T10:42:46,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/B/3615777c8aed4c0da3ffbf1b270c4a90 is 50, key is test_row_0/B:col10/1733740965316/Put/seqid=0 2024-12-09T10:42:46,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742284_1460 (size=12151) 2024-12-09T10:42:47,166 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:47,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59112 deadline: 1733741027160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:47,167 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:47,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59136 deadline: 1733741027160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:47,167 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:47,167 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:47,167 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:47,167 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:47,167 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:47,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59126 deadline: 1733741027160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:47,167 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:47,167 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:47,168 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:47,168 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:47,168 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:47,181 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:47,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59156 deadline: 1733741027169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:47,182 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:47,182 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:47,182 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:47,282 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/B/3615777c8aed4c0da3ffbf1b270c4a90 2024-12-09T10:42:47,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/C/56f1394847c84e6eb3ba5f5be953e616 is 50, key is test_row_0/C:col10/1733740965316/Put/seqid=0 2024-12-09T10:42:47,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742285_1461 (size=12151) 2024-12-09T10:42:47,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-09T10:42:47,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:47,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59114 deadline: 1733741027519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:47,528 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 7ea43b39fc24,36101,1733740827678 for 'test_row_2' in a9d4962ae8c180b5b7fa587448872d04 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2209 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T10:42:47,529 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:47,529 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:47,529 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:47,685 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:47,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59112 deadline: 1733741027680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:47,685 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:47,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59136 deadline: 1733741027680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:47,685 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:47,685 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:47,685 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:47,685 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:47,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59126 deadline: 1733741027680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:47,685 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:47,686 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:47,686 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:47,686 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:47,686 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:47,686 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:47,693 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/C/56f1394847c84e6eb3ba5f5be953e616 2024-12-09T10:42:47,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/A/58304220a6644c908c5a8c77acbe05d8 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/58304220a6644c908c5a8c77acbe05d8 2024-12-09T10:42:47,698 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:47,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59156 deadline: 1733741027689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:47,698 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:47,698 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:47,698 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:47,701 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/58304220a6644c908c5a8c77acbe05d8, entries=150, sequenceid=200, filesize=11.9 K 2024-12-09T10:42:47,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/B/3615777c8aed4c0da3ffbf1b270c4a90 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/3615777c8aed4c0da3ffbf1b270c4a90 2024-12-09T10:42:47,705 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/3615777c8aed4c0da3ffbf1b270c4a90, entries=150, sequenceid=200, filesize=11.9 K 2024-12-09T10:42:47,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/C/56f1394847c84e6eb3ba5f5be953e616 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/56f1394847c84e6eb3ba5f5be953e616 2024-12-09T10:42:47,708 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/56f1394847c84e6eb3ba5f5be953e616, entries=150, sequenceid=200, filesize=11.9 K 2024-12-09T10:42:47,709 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for a9d4962ae8c180b5b7fa587448872d04 in 1250ms, sequenceid=200, compaction requested=true 2024-12-09T10:42:47,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for a9d4962ae8c180b5b7fa587448872d04: 2024-12-09T10:42:47,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. 2024-12-09T10:42:47,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-12-09T10:42:47,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-12-09T10:42:47,712 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-12-09T10:42:47,712 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4770 sec 2024-12-09T10:42:47,713 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 2.4810 sec 2024-12-09T10:42:48,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(8855): Flush requested on a9d4962ae8c180b5b7fa587448872d04 2024-12-09T10:42:48,695 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a9d4962ae8c180b5b7fa587448872d04 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-09T10:42:48,695 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9d4962ae8c180b5b7fa587448872d04, store=A 2024-12-09T10:42:48,695 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:42:48,695 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9d4962ae8c180b5b7fa587448872d04, store=B 2024-12-09T10:42:48,695 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:42:48,695 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9d4962ae8c180b5b7fa587448872d04, store=C 2024-12-09T10:42:48,695 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:42:48,699 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/A/512c59480a7f49ee92e3b82a36856686 is 50, key is test_row_0/A:col10/1733740968693/Put/seqid=0 2024-12-09T10:42:48,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742286_1462 (size=14541) 2024-12-09T10:42:48,708 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/A/512c59480a7f49ee92e3b82a36856686 2024-12-09T10:42:48,720 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/B/a781c4afec684d37bf0a336c88e9d515 is 50, key is test_row_0/B:col10/1733740968693/Put/seqid=0 2024-12-09T10:42:48,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742287_1463 (size=12151) 2024-12-09T10:42:48,756 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:48,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59112 deadline: 1733741028741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:48,757 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:48,757 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:48,757 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:48,758 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733740968739, max timestamp=1733740968754Num uniques 859; 2024-12-09T10:42:48,758 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T10:42:48,758 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733740968739, max timestamp=1733740968754Num uniques 859; 2024-12-09T10:42:48,759 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733740968739, max timestamp=1733740968754Num uniques 859; 2024-12-09T10:42:48,759 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T10:42:48,759 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T10:42:48,817 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:48,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59126 deadline: 1733741028804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:48,818 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:48,818 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:48,818 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:48,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:48,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59136 deadline: 1733741028818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:48,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:48,819 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:48,819 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:48,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59156 deadline: 1733741028818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:48,819 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:48,820 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:48,820 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:48,820 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:48,868 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:48,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59112 deadline: 1733741028860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:48,869 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:48,869 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:48,869 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:48,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:48,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59126 deadline: 1733741028929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:48,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:48,941 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:48,941 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:48,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59136 deadline: 1733741028929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:48,941 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:48,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:48,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59156 deadline: 1733741028930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:48,941 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:48,942 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:48,942 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:48,942 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:48,942 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:48,942 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:49,087 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:49,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59112 deadline: 1733741029080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:49,087 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:49,087 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:49,087 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:49,133 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/B/a781c4afec684d37bf0a336c88e9d515 2024-12-09T10:42:49,141 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/C/9fb6f147b0ad44d493a159ebd2c70d92 is 50, key is test_row_0/C:col10/1733740968693/Put/seqid=0 2024-12-09T10:42:49,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742288_1464 (size=12151) 2024-12-09T10:42:49,162 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:49,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59126 deadline: 1733741029150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:49,163 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:49,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59156 deadline: 1733741029150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:49,163 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:49,163 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:49,163 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:49,163 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:49,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59136 deadline: 1733741029150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:49,163 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:49,163 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:49,163 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:49,163 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:49,163 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:49,163 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:49,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-09T10:42:49,390 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T10:42:49,391 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-09T10:42:49,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-12-09T10:42:49,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-09T10:42:49,392 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T10:42:49,393 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T10:42:49,393 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T10:42:49,411 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:49,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59112 deadline: 1733741029399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:49,412 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:49,412 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:49,412 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:49,481 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:49,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59126 deadline: 1733741029470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:49,481 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:49,481 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:49,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59156 deadline: 1733741029470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:49,481 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:49,481 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:49,481 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:49,482 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:49,482 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:49,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59136 deadline: 1733741029470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:49,482 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:49,482 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:49,482 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:49,482 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:49,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-09T10:42:49,545 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-09T10:42:49,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. 2024-12-09T10:42:49,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. as already flushing 2024-12-09T10:42:49,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. 2024-12-09T10:42:49,545 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:49,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:49,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:49,553 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/C/9fb6f147b0ad44d493a159ebd2c70d92 2024-12-09T10:42:49,558 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/A/512c59480a7f49ee92e3b82a36856686 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/512c59480a7f49ee92e3b82a36856686 2024-12-09T10:42:49,560 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:49,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59114 deadline: 1733741029549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:49,561 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/512c59480a7f49ee92e3b82a36856686, entries=200, sequenceid=213, filesize=14.2 K 2024-12-09T10:42:49,561 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 7ea43b39fc24,36101,1733740827678 for 'test_row_2' in a9d4962ae8c180b5b7fa587448872d04 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4242 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T10:42:49,561 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:49,561 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:49,561 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:49,562 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/B/a781c4afec684d37bf0a336c88e9d515 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/a781c4afec684d37bf0a336c88e9d515 2024-12-09T10:42:49,564 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/a781c4afec684d37bf0a336c88e9d515, entries=150, sequenceid=213, filesize=11.9 K 2024-12-09T10:42:49,565 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/C/9fb6f147b0ad44d493a159ebd2c70d92 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/9fb6f147b0ad44d493a159ebd2c70d92 2024-12-09T10:42:49,568 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/9fb6f147b0ad44d493a159ebd2c70d92, entries=150, sequenceid=213, filesize=11.9 K 2024-12-09T10:42:49,569 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for a9d4962ae8c180b5b7fa587448872d04 in 875ms, sequenceid=213, compaction requested=true 2024-12-09T10:42:49,569 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a9d4962ae8c180b5b7fa587448872d04: 2024-12-09T10:42:49,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9d4962ae8c180b5b7fa587448872d04:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T10:42:49,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:42:49,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9d4962ae8c180b5b7fa587448872d04:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T10:42:49,569 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T10:42:49,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:42:49,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9d4962ae8c180b5b7fa587448872d04:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T10:42:49,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T10:42:49,569 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T10:42:49,570 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T10:42:49,570 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39185 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T10:42:49,570 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1541): a9d4962ae8c180b5b7fa587448872d04/A is initiating minor compaction (all files) 2024-12-09T10:42:49,570 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1541): a9d4962ae8c180b5b7fa587448872d04/B is initiating minor compaction (all files) 2024-12-09T10:42:49,570 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a9d4962ae8c180b5b7fa587448872d04/A in TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. 2024-12-09T10:42:49,570 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a9d4962ae8c180b5b7fa587448872d04/B in TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. 2024-12-09T10:42:49,570 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/9bb012ad13804bf7a0596b186ed951af, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/58304220a6644c908c5a8c77acbe05d8, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/512c59480a7f49ee92e3b82a36856686] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp, totalSize=38.3 K 2024-12-09T10:42:49,570 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/0590e692e7154225a6985b40d5133ae8, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/3615777c8aed4c0da3ffbf1b270c4a90, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/a781c4afec684d37bf0a336c88e9d515] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp, totalSize=35.9 K 2024-12-09T10:42:49,570 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9bb012ad13804bf7a0596b186ed951af, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1733740964095 2024-12-09T10:42:49,570 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 0590e692e7154225a6985b40d5133ae8, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1733740964095 2024-12-09T10:42:49,571 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 3615777c8aed4c0da3ffbf1b270c4a90, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1733740965315 2024-12-09T10:42:49,571 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 58304220a6644c908c5a8c77acbe05d8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1733740965315 2024-12-09T10:42:49,571 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting a781c4afec684d37bf0a336c88e9d515, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1733740966507 2024-12-09T10:42:49,571 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 512c59480a7f49ee92e3b82a36856686, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1733740966506 2024-12-09T10:42:49,578 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a9d4962ae8c180b5b7fa587448872d04#B#compaction#392 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:42:49,579 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/B/24f011f67790402e942e1cb09ee3b373 is 50, key is test_row_0/B:col10/1733740968693/Put/seqid=0 2024-12-09T10:42:49,581 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a9d4962ae8c180b5b7fa587448872d04#A#compaction#393 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:42:49,582 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/A/dd96c1ed5f474d1f9d890acd3e6c5bc3 is 50, key is test_row_0/A:col10/1733740968693/Put/seqid=0 2024-12-09T10:42:49,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742289_1465 (size=12595) 2024-12-09T10:42:49,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742290_1466 (size=12595) 2024-12-09T10:42:49,608 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/A/dd96c1ed5f474d1f9d890acd3e6c5bc3 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/dd96c1ed5f474d1f9d890acd3e6c5bc3 2024-12-09T10:42:49,612 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a9d4962ae8c180b5b7fa587448872d04/A of a9d4962ae8c180b5b7fa587448872d04 into dd96c1ed5f474d1f9d890acd3e6c5bc3(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:42:49,612 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a9d4962ae8c180b5b7fa587448872d04: 2024-12-09T10:42:49,612 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., storeName=a9d4962ae8c180b5b7fa587448872d04/A, priority=13, startTime=1733740969569; duration=0sec 2024-12-09T10:42:49,612 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T10:42:49,612 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9d4962ae8c180b5b7fa587448872d04:A 2024-12-09T10:42:49,612 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T10:42:49,613 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46484 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T10:42:49,613 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1541): a9d4962ae8c180b5b7fa587448872d04/C is initiating minor compaction (all files) 2024-12-09T10:42:49,613 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a9d4962ae8c180b5b7fa587448872d04/C in TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. 2024-12-09T10:42:49,613 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/c1c5efacdb2f46b9bfec1e38e4c0fc6e, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/fe66cb496d904e0da0d60997990f353b, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/56f1394847c84e6eb3ba5f5be953e616, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/9fb6f147b0ad44d493a159ebd2c70d92] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp, totalSize=45.4 K 2024-12-09T10:42:49,613 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting c1c5efacdb2f46b9bfec1e38e4c0fc6e, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1733740964092 2024-12-09T10:42:49,614 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting fe66cb496d904e0da0d60997990f353b, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1733740965192 2024-12-09T10:42:49,614 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 56f1394847c84e6eb3ba5f5be953e616, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1733740965315 2024-12-09T10:42:49,614 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9fb6f147b0ad44d493a159ebd2c70d92, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1733740966507 2024-12-09T10:42:49,622 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a9d4962ae8c180b5b7fa587448872d04#C#compaction#394 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:42:49,623 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/C/9dc3c216f0264622824688e3f1366272 is 50, key is test_row_0/C:col10/1733740968693/Put/seqid=0 2024-12-09T10:42:49,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742291_1467 (size=12561) 2024-12-09T10:42:49,698 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-09T10:42:49,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. 2024-12-09T10:42:49,698 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing a9d4962ae8c180b5b7fa587448872d04 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-12-09T10:42:49,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9d4962ae8c180b5b7fa587448872d04, store=A 2024-12-09T10:42:49,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T10:42:49,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9d4962ae8c180b5b7fa587448872d04, store=B 2024-12-09T10:42:49,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T10:42:49,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9d4962ae8c180b5b7fa587448872d04, store=C 2024-12-09T10:42:49,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T10:42:49,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/A/43c06e50d8ec4d1aaf6070bf893b7331 is 50, key is test_row_0/A:col10/1733740968816/Put/seqid=0 2024-12-09T10:42:49,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742292_1468 (size=12151) 2024-12-09T10:42:49,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-09T10:42:49,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(8855): Flush requested on a9d4962ae8c180b5b7fa587448872d04 2024-12-09T10:42:49,924 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. as already flushing 2024-12-09T10:42:49,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:49,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59112 deadline: 1733741029981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:49,991 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:49,991 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:49,991 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:49,993 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/B/24f011f67790402e942e1cb09ee3b373 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/24f011f67790402e942e1cb09ee3b373 2024-12-09T10:42:49,997 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a9d4962ae8c180b5b7fa587448872d04/B of a9d4962ae8c180b5b7fa587448872d04 into 24f011f67790402e942e1cb09ee3b373(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:42:49,997 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a9d4962ae8c180b5b7fa587448872d04: 2024-12-09T10:42:49,997 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., storeName=a9d4962ae8c180b5b7fa587448872d04/B, priority=13, startTime=1733740969569; duration=0sec 2024-12-09T10:42:49,997 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:42:49,997 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9d4962ae8c180b5b7fa587448872d04:B 2024-12-09T10:42:50,000 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:50,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59126 deadline: 1733741029989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:50,001 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:50,001 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:50,001 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:50,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59156 deadline: 1733741029990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:50,001 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:50,001 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:50,001 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:50,001 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:50,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59136 deadline: 1733741029990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:50,001 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:50,001 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:50,001 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:50,001 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:50,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-09T10:42:50,030 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/C/9dc3c216f0264622824688e3f1366272 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/9dc3c216f0264622824688e3f1366272 2024-12-09T10:42:50,034 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a9d4962ae8c180b5b7fa587448872d04/C of a9d4962ae8c180b5b7fa587448872d04 into 9dc3c216f0264622824688e3f1366272(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:42:50,034 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a9d4962ae8c180b5b7fa587448872d04: 2024-12-09T10:42:50,034 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., storeName=a9d4962ae8c180b5b7fa587448872d04/C, priority=12, startTime=1733740969569; duration=0sec 2024-12-09T10:42:50,034 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:42:50,034 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9d4962ae8c180b5b7fa587448872d04:C 2024-12-09T10:42:50,102 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:50,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59112 deadline: 1733741030099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:50,103 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:50,103 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:50,103 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:50,110 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/A/43c06e50d8ec4d1aaf6070bf893b7331 2024-12-09T10:42:50,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/B/677fcc713ff14dee891a99f14b93868d is 50, key is test_row_0/B:col10/1733740968816/Put/seqid=0 2024-12-09T10:42:50,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742293_1469 (size=12151) 2024-12-09T10:42:50,312 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:50,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59112 deadline: 1733741030310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:50,313 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:50,313 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:50,313 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:50,529 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/B/677fcc713ff14dee891a99f14b93868d 2024-12-09T10:42:50,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-09T10:42:50,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/C/2269591a633c4010baf8548207af79b0 is 50, key is test_row_0/C:col10/1733740968816/Put/seqid=0 2024-12-09T10:42:50,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742294_1470 (size=12151) 2024-12-09T10:42:50,624 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:50,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59112 deadline: 1733741030620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:50,625 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:50,625 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:50,625 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:50,940 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/C/2269591a633c4010baf8548207af79b0 2024-12-09T10:42:50,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/A/43c06e50d8ec4d1aaf6070bf893b7331 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/43c06e50d8ec4d1aaf6070bf893b7331 2024-12-09T10:42:50,947 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/43c06e50d8ec4d1aaf6070bf893b7331, entries=150, sequenceid=249, filesize=11.9 K 2024-12-09T10:42:50,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/B/677fcc713ff14dee891a99f14b93868d as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/677fcc713ff14dee891a99f14b93868d 2024-12-09T10:42:50,951 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/677fcc713ff14dee891a99f14b93868d, entries=150, sequenceid=249, filesize=11.9 K 2024-12-09T10:42:50,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/C/2269591a633c4010baf8548207af79b0 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/2269591a633c4010baf8548207af79b0 2024-12-09T10:42:50,955 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/2269591a633c4010baf8548207af79b0, entries=150, sequenceid=249, filesize=11.9 K 2024-12-09T10:42:50,956 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for a9d4962ae8c180b5b7fa587448872d04 in 1258ms, sequenceid=249, compaction requested=false 2024-12-09T10:42:50,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for a9d4962ae8c180b5b7fa587448872d04: 2024-12-09T10:42:50,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. 2024-12-09T10:42:50,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-12-09T10:42:50,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-12-09T10:42:50,958 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-12-09T10:42:50,958 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5640 sec 2024-12-09T10:42:50,960 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 1.5680 sec 2024-12-09T10:42:51,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(8855): Flush requested on a9d4962ae8c180b5b7fa587448872d04 2024-12-09T10:42:51,014 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a9d4962ae8c180b5b7fa587448872d04 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-09T10:42:51,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9d4962ae8c180b5b7fa587448872d04, store=A 2024-12-09T10:42:51,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:42:51,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9d4962ae8c180b5b7fa587448872d04, store=B 2024-12-09T10:42:51,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:42:51,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9d4962ae8c180b5b7fa587448872d04, store=C 2024-12-09T10:42:51,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:42:51,018 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/A/c7729feaaba04e168cd44303728f3c7c is 50, key is test_row_0/A:col10/1733740971012/Put/seqid=0 2024-12-09T10:42:51,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742295_1471 (size=12251) 2024-12-09T10:42:51,117 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:51,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59156 deadline: 1733741031104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:51,117 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:51,117 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:51,117 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:51,120 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:51,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59126 deadline: 1733741031104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:51,120 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:51,120 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:51,120 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:51,130 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:51,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59136 deadline: 1733741031117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:51,130 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:51,130 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:51,130 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:51,137 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:51,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59112 deadline: 1733741031129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:51,138 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:51,138 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:51,138 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:51,228 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:51,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59156 deadline: 1733741031220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:51,229 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:51,229 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:51,229 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:51,237 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:51,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59126 deadline: 1733741031229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:51,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:51,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:51,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:51,250 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:51,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59136 deadline: 1733741031239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:51,251 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:51,251 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:51,251 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:51,424 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/A/c7729feaaba04e168cd44303728f3c7c 2024-12-09T10:42:51,430 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/B/88d2d65910ce43e4adee06171ab91f10 is 50, key is test_row_0/B:col10/1733740971012/Put/seqid=0 2024-12-09T10:42:51,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742296_1472 (size=12251) 2024-12-09T10:42:51,449 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:51,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59156 deadline: 1733741031440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:51,450 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:51,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59126 deadline: 1733741031440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:51,450 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:51,450 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:51,450 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:51,450 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:51,450 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:51,450 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:51,468 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:51,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59136 deadline: 1733741031459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:51,469 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:51,469 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:51,469 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:51,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-09T10:42:51,550 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T10:42:51,551 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-09T10:42:51,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-12-09T10:42:51,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-09T10:42:51,553 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T10:42:51,553 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T10:42:51,554 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T10:42:51,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-09T10:42:51,706 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-09T10:42:51,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. 2024-12-09T10:42:51,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. as already flushing 2024-12-09T10:42:51,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. 2024-12-09T10:42:51,706 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:51,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:51,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:51,767 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:51,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59156 deadline: 1733741031760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:51,767 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:51,767 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:51,767 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:51,767 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:51,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59126 deadline: 1733741031760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:51,768 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:51,768 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:51,768 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:51,784 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:51,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59136 deadline: 1733741031779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:51,784 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:51,784 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:51,784 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:51,834 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/B/88d2d65910ce43e4adee06171ab91f10 2024-12-09T10:42:51,841 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/C/27058d4d5b7d4e53b364c81a0ed2ab42 is 50, key is test_row_0/C:col10/1733740971012/Put/seqid=0 2024-12-09T10:42:51,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742297_1473 (size=12251) 2024-12-09T10:42:51,859 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-09T10:42:51,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. 2024-12-09T10:42:51,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. as already flushing 2024-12-09T10:42:51,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. 2024-12-09T10:42:51,859 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:51,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:51,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:51,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-09T10:42:52,012 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-09T10:42:52,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. 2024-12-09T10:42:52,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. as already flushing 2024-12-09T10:42:52,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. 2024-12-09T10:42:52,012 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:52,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:52,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:52,160 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:52,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59112 deadline: 1733741032150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:52,161 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 7ea43b39fc24,36101,1733740827678 for 'test_row_2' in a9d4962ae8c180b5b7fa587448872d04 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T10:42:52,162 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:52,162 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:52,162 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:52,165 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-09T10:42:52,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. 2024-12-09T10:42:52,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. as already flushing 2024-12-09T10:42:52,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. 2024-12-09T10:42:52,165 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:52,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:52,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:52,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-09T10:42:52,245 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/C/27058d4d5b7d4e53b364c81a0ed2ab42 2024-12-09T10:42:52,249 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/A/c7729feaaba04e168cd44303728f3c7c as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/c7729feaaba04e168cd44303728f3c7c 2024-12-09T10:42:52,252 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/c7729feaaba04e168cd44303728f3c7c, entries=150, sequenceid=263, filesize=12.0 K 2024-12-09T10:42:52,252 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/B/88d2d65910ce43e4adee06171ab91f10 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/88d2d65910ce43e4adee06171ab91f10 2024-12-09T10:42:52,259 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/88d2d65910ce43e4adee06171ab91f10, entries=150, sequenceid=263, filesize=12.0 K 2024-12-09T10:42:52,260 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/C/27058d4d5b7d4e53b364c81a0ed2ab42 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/27058d4d5b7d4e53b364c81a0ed2ab42 2024-12-09T10:42:52,263 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/27058d4d5b7d4e53b364c81a0ed2ab42, entries=150, sequenceid=263, filesize=12.0 K 2024-12-09T10:42:52,264 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for a9d4962ae8c180b5b7fa587448872d04 in 1250ms, sequenceid=263, compaction requested=true 2024-12-09T10:42:52,264 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a9d4962ae8c180b5b7fa587448872d04: 2024-12-09T10:42:52,264 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9d4962ae8c180b5b7fa587448872d04:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T10:42:52,264 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:42:52,264 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T10:42:52,264 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9d4962ae8c180b5b7fa587448872d04:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T10:42:52,264 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T10:42:52,264 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:42:52,264 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9d4962ae8c180b5b7fa587448872d04:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T10:42:52,264 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T10:42:52,266 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36997 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T10:42:52,266 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36997 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T10:42:52,266 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1541): a9d4962ae8c180b5b7fa587448872d04/B is initiating minor compaction (all files) 2024-12-09T10:42:52,266 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1541): a9d4962ae8c180b5b7fa587448872d04/A is initiating minor compaction (all files) 2024-12-09T10:42:52,266 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a9d4962ae8c180b5b7fa587448872d04/A in TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. 2024-12-09T10:42:52,266 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a9d4962ae8c180b5b7fa587448872d04/B in TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. 2024-12-09T10:42:52,266 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/dd96c1ed5f474d1f9d890acd3e6c5bc3, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/43c06e50d8ec4d1aaf6070bf893b7331, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/c7729feaaba04e168cd44303728f3c7c] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp, totalSize=36.1 K 2024-12-09T10:42:52,266 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/24f011f67790402e942e1cb09ee3b373, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/677fcc713ff14dee891a99f14b93868d, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/88d2d65910ce43e4adee06171ab91f10] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp, totalSize=36.1 K 2024-12-09T10:42:52,266 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting dd96c1ed5f474d1f9d890acd3e6c5bc3, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1733740966507 2024-12-09T10:42:52,266 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 24f011f67790402e942e1cb09ee3b373, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1733740966507 2024-12-09T10:42:52,266 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 677fcc713ff14dee891a99f14b93868d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1733740968800 2024-12-09T10:42:52,266 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 43c06e50d8ec4d1aaf6070bf893b7331, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1733740968800 2024-12-09T10:42:52,267 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 88d2d65910ce43e4adee06171ab91f10, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1733740969939 2024-12-09T10:42:52,267 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting c7729feaaba04e168cd44303728f3c7c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1733740969939 2024-12-09T10:42:52,273 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a9d4962ae8c180b5b7fa587448872d04#A#compaction#401 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:42:52,273 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/A/166732208ef842fa83e58177ff49cbed is 50, key is test_row_0/A:col10/1733740971012/Put/seqid=0 2024-12-09T10:42:52,275 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a9d4962ae8c180b5b7fa587448872d04#B#compaction#402 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:42:52,276 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/B/6a0e0a4648504a47bd647515526435ad is 50, key is test_row_0/B:col10/1733740971012/Put/seqid=0 2024-12-09T10:42:52,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742298_1474 (size=12797) 2024-12-09T10:42:52,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(8855): Flush requested on a9d4962ae8c180b5b7fa587448872d04 2024-12-09T10:42:52,287 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a9d4962ae8c180b5b7fa587448872d04 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-09T10:42:52,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9d4962ae8c180b5b7fa587448872d04, store=A 2024-12-09T10:42:52,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T10:42:52,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9d4962ae8c180b5b7fa587448872d04, store=B 2024-12-09T10:42:52,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T10:42:52,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9d4962ae8c180b5b7fa587448872d04, store=C 2024-12-09T10:42:52,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T10:42:52,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742299_1475 (size=12797) 2024-12-09T10:42:52,294 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/A/761a2ab0f27f4743be169602f9bfe798 is 50, key is test_row_0/A:col10/1733740972286/Put/seqid=0 2024-12-09T10:42:52,297 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/A/166732208ef842fa83e58177ff49cbed as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/166732208ef842fa83e58177ff49cbed 2024-12-09T10:42:52,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742300_1476 (size=14741) 2024-12-09T10:42:52,301 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a9d4962ae8c180b5b7fa587448872d04/A of a9d4962ae8c180b5b7fa587448872d04 into 166732208ef842fa83e58177ff49cbed(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:42:52,301 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a9d4962ae8c180b5b7fa587448872d04: 2024-12-09T10:42:52,301 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., storeName=a9d4962ae8c180b5b7fa587448872d04/A, priority=13, startTime=1733740972264; duration=0sec 2024-12-09T10:42:52,301 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T10:42:52,301 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9d4962ae8c180b5b7fa587448872d04:A 2024-12-09T10:42:52,301 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T10:42:52,302 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36963 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T10:42:52,302 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1541): a9d4962ae8c180b5b7fa587448872d04/C is initiating minor compaction (all files) 2024-12-09T10:42:52,302 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a9d4962ae8c180b5b7fa587448872d04/C in TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. 2024-12-09T10:42:52,303 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/9dc3c216f0264622824688e3f1366272, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/2269591a633c4010baf8548207af79b0, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/27058d4d5b7d4e53b364c81a0ed2ab42] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp, totalSize=36.1 K 2024-12-09T10:42:52,303 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9dc3c216f0264622824688e3f1366272, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1733740966507 2024-12-09T10:42:52,303 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2269591a633c4010baf8548207af79b0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1733740968800 2024-12-09T10:42:52,303 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 27058d4d5b7d4e53b364c81a0ed2ab42, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1733740969939 2024-12-09T10:42:52,310 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a9d4962ae8c180b5b7fa587448872d04#C#compaction#404 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:42:52,311 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/C/71084c55670644f3afa26a3a496798ed is 50, key is test_row_0/C:col10/1733740971012/Put/seqid=0 2024-12-09T10:42:52,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742301_1477 (size=12763) 2024-12-09T10:42:52,318 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-09T10:42:52,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. 2024-12-09T10:42:52,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. as already flushing 2024-12-09T10:42:52,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. 2024-12-09T10:42:52,319 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:52,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:52,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:52,321 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:52,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59136 deadline: 1733741032318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:52,321 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:52,321 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:52,321 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:52,328 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:52,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59126 deadline: 1733741032320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:52,329 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:52,329 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:52,329 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:52,334 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:52,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59156 deadline: 1733741032321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:52,334 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:52,335 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:52,335 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:52,439 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:52,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59136 deadline: 1733741032430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:52,440 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:52,440 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:52,440 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:52,445 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:52,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59126 deadline: 1733741032439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:52,446 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:52,446 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:52,446 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:52,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59156 deadline: 1733741032439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:52,446 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:52,446 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:52,446 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:52,446 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:52,470 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-09T10:42:52,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. 2024-12-09T10:42:52,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. as already flushing 2024-12-09T10:42:52,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. 2024-12-09T10:42:52,471 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:52,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:52,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:52,623 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-09T10:42:52,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. 2024-12-09T10:42:52,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. as already flushing 2024-12-09T10:42:52,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. 2024-12-09T10:42:52,624 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:52,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:52,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:52,654 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:52,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59136 deadline: 1733741032650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:52,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:52,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59126 deadline: 1733741032650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:52,655 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:52,655 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:52,655 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:52,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:52,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59156 deadline: 1733741032650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:52,655 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:52,655 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:52,655 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:52,655 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:52,656 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:52,656 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:52,690 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/B/6a0e0a4648504a47bd647515526435ad as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/6a0e0a4648504a47bd647515526435ad 2024-12-09T10:42:52,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-09T10:42:52,694 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a9d4962ae8c180b5b7fa587448872d04/B of a9d4962ae8c180b5b7fa587448872d04 into 6a0e0a4648504a47bd647515526435ad(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:42:52,694 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a9d4962ae8c180b5b7fa587448872d04: 2024-12-09T10:42:52,694 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., storeName=a9d4962ae8c180b5b7fa587448872d04/B, priority=13, startTime=1733740972264; duration=0sec 2024-12-09T10:42:52,694 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:42:52,694 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9d4962ae8c180b5b7fa587448872d04:B 2024-12-09T10:42:52,698 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/A/761a2ab0f27f4743be169602f9bfe798 2024-12-09T10:42:52,705 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/B/b1077b97e6c849ec938ca2979362a2db is 50, key is test_row_0/B:col10/1733740972286/Put/seqid=0 2024-12-09T10:42:52,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742302_1478 (size=12301) 2024-12-09T10:42:52,723 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/C/71084c55670644f3afa26a3a496798ed as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/71084c55670644f3afa26a3a496798ed 2024-12-09T10:42:52,727 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a9d4962ae8c180b5b7fa587448872d04/C of a9d4962ae8c180b5b7fa587448872d04 into 71084c55670644f3afa26a3a496798ed(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:42:52,727 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a9d4962ae8c180b5b7fa587448872d04: 2024-12-09T10:42:52,727 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., storeName=a9d4962ae8c180b5b7fa587448872d04/C, priority=13, startTime=1733740972264; duration=0sec 2024-12-09T10:42:52,727 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:42:52,727 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9d4962ae8c180b5b7fa587448872d04:C 2024-12-09T10:42:52,776 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-09T10:42:52,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. 2024-12-09T10:42:52,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. as already flushing 2024-12-09T10:42:52,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. 2024-12-09T10:42:52,777 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:52,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:52,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:52,928 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-09T10:42:52,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. 2024-12-09T10:42:52,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. as already flushing 2024-12-09T10:42:52,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. 2024-12-09T10:42:52,929 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:52,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:52,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:52,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:52,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59136 deadline: 1733741032960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:52,964 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:52,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59126 deadline: 1733741032960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:52,964 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:52,964 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:52,964 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:52,964 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:52,964 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:52,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59156 deadline: 1733741032960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:52,964 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:52,964 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:52,964 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:52,964 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:52,964 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:53,081 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-09T10:42:53,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. 2024-12-09T10:42:53,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. as already flushing 2024-12-09T10:42:53,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. 2024-12-09T10:42:53,082 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:53,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:53,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:53,109 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/B/b1077b97e6c849ec938ca2979362a2db 2024-12-09T10:42:53,117 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/C/7f0074694757496c8a31bdfcd7bfb026 is 50, key is test_row_0/C:col10/1733740972286/Put/seqid=0 2024-12-09T10:42:53,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742303_1479 (size=12301) 2024-12-09T10:42:53,121 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/C/7f0074694757496c8a31bdfcd7bfb026 2024-12-09T10:42:53,125 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/A/761a2ab0f27f4743be169602f9bfe798 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/761a2ab0f27f4743be169602f9bfe798 2024-12-09T10:42:53,128 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/761a2ab0f27f4743be169602f9bfe798, entries=200, sequenceid=284, filesize=14.4 K 2024-12-09T10:42:53,129 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/B/b1077b97e6c849ec938ca2979362a2db as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/b1077b97e6c849ec938ca2979362a2db 2024-12-09T10:42:53,134 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/b1077b97e6c849ec938ca2979362a2db, entries=150, sequenceid=284, filesize=12.0 K 2024-12-09T10:42:53,135 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/C/7f0074694757496c8a31bdfcd7bfb026 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/7f0074694757496c8a31bdfcd7bfb026 2024-12-09T10:42:53,138 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/7f0074694757496c8a31bdfcd7bfb026, entries=150, sequenceid=284, filesize=12.0 K 2024-12-09T10:42:53,139 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for a9d4962ae8c180b5b7fa587448872d04 in 852ms, sequenceid=284, compaction requested=false 2024-12-09T10:42:53,139 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a9d4962ae8c180b5b7fa587448872d04: 2024-12-09T10:42:53,234 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-09T10:42:53,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. 2024-12-09T10:42:53,235 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing a9d4962ae8c180b5b7fa587448872d04 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-09T10:42:53,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9d4962ae8c180b5b7fa587448872d04, store=A 2024-12-09T10:42:53,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:42:53,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9d4962ae8c180b5b7fa587448872d04, store=B 2024-12-09T10:42:53,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:42:53,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9d4962ae8c180b5b7fa587448872d04, store=C 2024-12-09T10:42:53,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:42:53,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/A/1633ce36d61f4331ba8f98e701fade04 is 50, key is test_row_0/A:col10/1733740972317/Put/seqid=0 2024-12-09T10:42:53,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742304_1480 (size=12301) 2024-12-09T10:42:53,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(8855): Flush requested on a9d4962ae8c180b5b7fa587448872d04 2024-12-09T10:42:53,474 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. as already flushing 2024-12-09T10:42:53,536 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:53,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59156 deadline: 1733741033523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:53,536 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733740973500, max timestamp=1733740973532Num uniques 859; 2024-12-09T10:42:53,536 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:53,536 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T10:42:53,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59136 deadline: 1733741033528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:53,536 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733740973500, max timestamp=1733740973532Num uniques 859; 2024-12-09T10:42:53,536 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:53,536 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:53,536 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T10:42:53,536 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:53,536 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733740973500, max timestamp=1733740973532Num uniques 859; 2024-12-09T10:42:53,536 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T10:42:53,537 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:53,537 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:53,537 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:53,644 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/A/1633ce36d61f4331ba8f98e701fade04 2024-12-09T10:42:53,649 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:53,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59156 deadline: 1733741033640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:53,650 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:53,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:53,650 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:53,650 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:53,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59136 deadline: 1733741033640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:53,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/B/b0529cfaae2d4ecb918091868205ac55 is 50, key is test_row_0/B:col10/1733740972317/Put/seqid=0 2024-12-09T10:42:53,650 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:53,650 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:53,650 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:53,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742305_1481 (size=12301) 2024-12-09T10:42:53,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:53,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59126 deadline: 1733741033642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:53,657 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:53,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59114 deadline: 1733741033643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:53,658 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:53,658 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:53,658 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:53,658 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:53,658 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:53,658 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:53,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-09T10:42:53,759 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:53,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59126 deadline: 1733741033759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:53,760 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:53,760 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:53,760 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:53,775 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:53,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59114 deadline: 1733741033769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:53,776 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:53,776 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:53,776 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:53,864 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:53,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59156 deadline: 1733741033859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:53,864 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:53,864 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:53,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59136 deadline: 1733741033859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:53,864 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:53,864 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:53,864 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:53,865 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:53,865 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:53,982 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:53,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59126 deadline: 1733741033969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:53,983 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:53,983 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:53,983 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:53,984 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:53,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59114 deadline: 1733741033980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:53,984 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:53,984 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:53,984 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:54,054 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/B/b0529cfaae2d4ecb918091868205ac55 2024-12-09T10:42:54,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/C/d87cc844c4ab4030aa8f1985aa864598 is 50, key is test_row_0/C:col10/1733740972317/Put/seqid=0 2024-12-09T10:42:54,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742306_1482 (size=12301) 2024-12-09T10:42:54,173 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:54,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59156 deadline: 1733741034169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:54,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:54,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59136 deadline: 1733741034169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:54,174 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:54,174 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:54,174 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:54,174 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:54,174 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:54,174 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:54,190 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:54,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59112 deadline: 1733741034180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:54,191 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 7ea43b39fc24,36101,1733740827678 for 'test_row_2' in a9d4962ae8c180b5b7fa587448872d04 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4209 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T10:42:54,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:54,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:54,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:54,304 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:54,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59126 deadline: 1733741034290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:54,304 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:54,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59114 deadline: 1733741034290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:54,304 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:54,305 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:54,305 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:54,305 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:54,305 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:54,305 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:54,465 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/C/d87cc844c4ab4030aa8f1985aa864598 2024-12-09T10:42:54,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/A/1633ce36d61f4331ba8f98e701fade04 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/1633ce36d61f4331ba8f98e701fade04 2024-12-09T10:42:54,473 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/1633ce36d61f4331ba8f98e701fade04, entries=150, sequenceid=298, filesize=12.0 K 2024-12-09T10:42:54,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/B/b0529cfaae2d4ecb918091868205ac55 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/b0529cfaae2d4ecb918091868205ac55 2024-12-09T10:42:54,477 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/b0529cfaae2d4ecb918091868205ac55, entries=150, sequenceid=298, filesize=12.0 K 2024-12-09T10:42:54,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/C/d87cc844c4ab4030aa8f1985aa864598 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/d87cc844c4ab4030aa8f1985aa864598 2024-12-09T10:42:54,480 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/d87cc844c4ab4030aa8f1985aa864598, entries=150, sequenceid=298, filesize=12.0 K 2024-12-09T10:42:54,482 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for a9d4962ae8c180b5b7fa587448872d04 in 1247ms, sequenceid=298, compaction requested=true 2024-12-09T10:42:54,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for a9d4962ae8c180b5b7fa587448872d04: 2024-12-09T10:42:54,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. 2024-12-09T10:42:54,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-12-09T10:42:54,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-12-09T10:42:54,485 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-12-09T10:42:54,485 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9290 sec 2024-12-09T10:42:54,487 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 2.9350 sec 2024-12-09T10:42:54,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(8855): Flush requested on a9d4962ae8c180b5b7fa587448872d04 2024-12-09T10:42:54,687 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a9d4962ae8c180b5b7fa587448872d04 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-12-09T10:42:54,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9d4962ae8c180b5b7fa587448872d04, store=A 2024-12-09T10:42:54,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T10:42:54,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9d4962ae8c180b5b7fa587448872d04, store=B 2024-12-09T10:42:54,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T10:42:54,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9d4962ae8c180b5b7fa587448872d04, store=C 2024-12-09T10:42:54,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T10:42:54,692 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/A/eff44505efda40b2bc47227d510dc082 is 50, key is test_row_0/A:col10/1733740973592/Put/seqid=0 2024-12-09T10:42:54,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742307_1483 (size=14741) 2024-12-09T10:42:54,726 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:54,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59136 deadline: 1733741034721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:54,727 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:54,727 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:54,727 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:54,736 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:54,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59156 deadline: 1733741034722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:54,737 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:54,737 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:54,737 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:54,821 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:54,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59114 deadline: 1733741034809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:54,822 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:54,822 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:54,822 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:54,823 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:54,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59126 deadline: 1733741034819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:54,824 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:54,824 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:54,824 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:54,837 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:54,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59136 deadline: 1733741034830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:54,838 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:54,838 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:54,838 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:54,853 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:54,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59156 deadline: 1733741034839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:54,854 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:54,854 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:54,854 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:55,065 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:55,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59136 deadline: 1733741035050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:55,068 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:55,068 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:55,068 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:55,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:55,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59156 deadline: 1733741035060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:55,079 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:55,079 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:55,079 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:55,097 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/A/eff44505efda40b2bc47227d510dc082 2024-12-09T10:42:55,115 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/B/a520bba8420c40489c93573e721767a2 is 50, key is test_row_0/B:col10/1733740973592/Put/seqid=0 2024-12-09T10:42:55,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742308_1484 (size=12301) 2024-12-09T10:42:55,388 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:55,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59136 deadline: 1733741035379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:55,388 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:55,389 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:55,389 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:55,400 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:55,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59156 deadline: 1733741035390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:55,400 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:55,401 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:55,401 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:55,527 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/B/a520bba8420c40489c93573e721767a2 2024-12-09T10:42:55,535 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/C/0c8aa6099be54bcf898a31e41012af66 is 50, key is test_row_0/C:col10/1733740973592/Put/seqid=0 2024-12-09T10:42:55,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742309_1485 (size=12301) 2024-12-09T10:42:55,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-09T10:42:55,710 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T10:42:55,710 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-09T10:42:55,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-12-09T10:42:55,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-09T10:42:55,712 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T10:42:55,713 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T10:42:55,713 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T10:42:55,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-09T10:42:55,841 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:55,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59114 deadline: 1733741035830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:55,841 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 7ea43b39fc24,36101,1733740827678 for 'test_row_2' in a9d4962ae8c180b5b7fa587448872d04 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2198 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T10:42:55,842 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:55,842 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:55,842 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:55,851 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:55,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59126 deadline: 1733741035840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:55,852 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 7ea43b39fc24,36101,1733740827678 for 'test_row_1' in a9d4962ae8c180b5b7fa587448872d04 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2210 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T10:42:55,852 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:55,852 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:55,852 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:55,865 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-12-09T10:42:55,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. 2024-12-09T10:42:55,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. as already flushing 2024-12-09T10:42:55,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. 2024-12-09T10:42:55,866 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:55,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:55,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:55,912 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:55,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59136 deadline: 1733741035899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:55,913 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:55,913 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:55,913 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:55,920 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:55,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59156 deadline: 1733741035910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:55,920 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:55,920 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:55,920 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:55,934 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-09T10:42:55,939 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/C/0c8aa6099be54bcf898a31e41012af66 2024-12-09T10:42:55,943 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/A/eff44505efda40b2bc47227d510dc082 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/eff44505efda40b2bc47227d510dc082 2024-12-09T10:42:55,947 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/eff44505efda40b2bc47227d510dc082, entries=200, sequenceid=334, filesize=14.4 K 2024-12-09T10:42:55,947 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/B/a520bba8420c40489c93573e721767a2 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/a520bba8420c40489c93573e721767a2 2024-12-09T10:42:55,952 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/a520bba8420c40489c93573e721767a2, entries=150, sequenceid=334, filesize=12.0 K 2024-12-09T10:42:55,952 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/C/0c8aa6099be54bcf898a31e41012af66 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/0c8aa6099be54bcf898a31e41012af66 2024-12-09T10:42:55,956 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/0c8aa6099be54bcf898a31e41012af66, entries=150, sequenceid=334, filesize=12.0 K 2024-12-09T10:42:55,957 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for a9d4962ae8c180b5b7fa587448872d04 in 1270ms, sequenceid=334, compaction requested=true 2024-12-09T10:42:55,957 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a9d4962ae8c180b5b7fa587448872d04: 2024-12-09T10:42:55,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9d4962ae8c180b5b7fa587448872d04:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T10:42:55,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:42:55,957 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T10:42:55,957 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T10:42:55,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9d4962ae8c180b5b7fa587448872d04:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T10:42:55,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:42:55,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9d4962ae8c180b5b7fa587448872d04:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T10:42:55,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T10:42:55,958 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49700 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T10:42:55,958 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54580 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T10:42:55,958 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1541): a9d4962ae8c180b5b7fa587448872d04/B is initiating minor compaction (all files) 2024-12-09T10:42:55,958 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1541): a9d4962ae8c180b5b7fa587448872d04/A is initiating minor compaction (all files) 2024-12-09T10:42:55,958 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a9d4962ae8c180b5b7fa587448872d04/B in TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. 2024-12-09T10:42:55,958 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a9d4962ae8c180b5b7fa587448872d04/A in TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. 2024-12-09T10:42:55,958 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/6a0e0a4648504a47bd647515526435ad, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/b1077b97e6c849ec938ca2979362a2db, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/b0529cfaae2d4ecb918091868205ac55, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/a520bba8420c40489c93573e721767a2] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp, totalSize=48.5 K 2024-12-09T10:42:55,958 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/166732208ef842fa83e58177ff49cbed, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/761a2ab0f27f4743be169602f9bfe798, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/1633ce36d61f4331ba8f98e701fade04, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/eff44505efda40b2bc47227d510dc082] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp, totalSize=53.3 K 2024-12-09T10:42:55,959 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a0e0a4648504a47bd647515526435ad, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1733740969939 2024-12-09T10:42:55,959 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 166732208ef842fa83e58177ff49cbed, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1733740969939 2024-12-09T10:42:55,960 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 761a2ab0f27f4743be169602f9bfe798, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1733740971101 2024-12-09T10:42:55,960 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting b1077b97e6c849ec938ca2979362a2db, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1733740971101 2024-12-09T10:42:55,960 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1633ce36d61f4331ba8f98e701fade04, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1733740972300 2024-12-09T10:42:55,960 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting b0529cfaae2d4ecb918091868205ac55, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1733740972300 2024-12-09T10:42:55,960 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting eff44505efda40b2bc47227d510dc082, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1733740973592 2024-12-09T10:42:55,960 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting a520bba8420c40489c93573e721767a2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1733740973592 2024-12-09T10:42:55,969 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a9d4962ae8c180b5b7fa587448872d04#B#compaction#413 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:42:55,969 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/B/2dfb0722739a4698960871f1343a1285 is 50, key is test_row_0/B:col10/1733740973592/Put/seqid=0 2024-12-09T10:42:55,970 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a9d4962ae8c180b5b7fa587448872d04#A#compaction#414 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:42:55,970 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/A/db824540a2184f01b017efbf56bdfb90 is 50, key is test_row_0/A:col10/1733740973592/Put/seqid=0 2024-12-09T10:42:55,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742310_1486 (size=12983) 2024-12-09T10:42:55,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742311_1487 (size=12983) 2024-12-09T10:42:55,988 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/A/db824540a2184f01b017efbf56bdfb90 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/db824540a2184f01b017efbf56bdfb90 2024-12-09T10:42:55,991 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/B/2dfb0722739a4698960871f1343a1285 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/2dfb0722739a4698960871f1343a1285 2024-12-09T10:42:55,993 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a9d4962ae8c180b5b7fa587448872d04/A of a9d4962ae8c180b5b7fa587448872d04 into db824540a2184f01b017efbf56bdfb90(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:42:55,993 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a9d4962ae8c180b5b7fa587448872d04: 2024-12-09T10:42:55,993 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., storeName=a9d4962ae8c180b5b7fa587448872d04/A, priority=12, startTime=1733740975957; duration=0sec 2024-12-09T10:42:55,993 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T10:42:55,993 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9d4962ae8c180b5b7fa587448872d04:A 2024-12-09T10:42:55,993 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T10:42:55,996 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49666 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T10:42:55,996 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1541): a9d4962ae8c180b5b7fa587448872d04/C is initiating minor compaction (all files) 2024-12-09T10:42:55,996 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a9d4962ae8c180b5b7fa587448872d04/C in TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. 2024-12-09T10:42:55,996 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a9d4962ae8c180b5b7fa587448872d04/B of a9d4962ae8c180b5b7fa587448872d04 into 2dfb0722739a4698960871f1343a1285(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:42:55,996 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a9d4962ae8c180b5b7fa587448872d04: 2024-12-09T10:42:55,996 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., storeName=a9d4962ae8c180b5b7fa587448872d04/B, priority=12, startTime=1733740975957; duration=0sec 2024-12-09T10:42:55,996 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/71084c55670644f3afa26a3a496798ed, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/7f0074694757496c8a31bdfcd7bfb026, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/d87cc844c4ab4030aa8f1985aa864598, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/0c8aa6099be54bcf898a31e41012af66] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp, totalSize=48.5 K 2024-12-09T10:42:55,996 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:42:55,996 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9d4962ae8c180b5b7fa587448872d04:B 2024-12-09T10:42:55,996 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 71084c55670644f3afa26a3a496798ed, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1733740969939 2024-12-09T10:42:55,997 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f0074694757496c8a31bdfcd7bfb026, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1733740971101 2024-12-09T10:42:55,997 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting d87cc844c4ab4030aa8f1985aa864598, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1733740972300 2024-12-09T10:42:55,997 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0c8aa6099be54bcf898a31e41012af66, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1733740973592 2024-12-09T10:42:56,005 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a9d4962ae8c180b5b7fa587448872d04#C#compaction#415 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:42:56,006 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/C/b39d155fec7d439b9bf4a265f21d46c9 is 50, key is test_row_0/C:col10/1733740973592/Put/seqid=0 2024-12-09T10:42:56,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742312_1488 (size=12949) 2024-12-09T10:42:56,019 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-12-09T10:42:56,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. 2024-12-09T10:42:56,019 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing a9d4962ae8c180b5b7fa587448872d04 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-09T10:42:56,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9d4962ae8c180b5b7fa587448872d04, store=A 2024-12-09T10:42:56,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:42:56,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9d4962ae8c180b5b7fa587448872d04, store=B 2024-12-09T10:42:56,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:42:56,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9d4962ae8c180b5b7fa587448872d04, store=C 2024-12-09T10:42:56,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:42:56,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/A/eb72702febac4491aa661801539fbf6e is 50, key is test_row_0/A:col10/1733740974721/Put/seqid=0 2024-12-09T10:42:56,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-09T10:42:56,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742313_1489 (size=12301) 2024-12-09T10:42:56,033 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/A/eb72702febac4491aa661801539fbf6e 2024-12-09T10:42:56,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/B/3d08c03a384f4177a07ba356e4dda6d6 is 50, key is test_row_0/B:col10/1733740974721/Put/seqid=0 2024-12-09T10:42:56,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742314_1490 (size=12301) 2024-12-09T10:42:56,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-09T10:42:56,419 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/C/b39d155fec7d439b9bf4a265f21d46c9 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/b39d155fec7d439b9bf4a265f21d46c9 2024-12-09T10:42:56,424 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a9d4962ae8c180b5b7fa587448872d04/C of a9d4962ae8c180b5b7fa587448872d04 into b39d155fec7d439b9bf4a265f21d46c9(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:42:56,424 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a9d4962ae8c180b5b7fa587448872d04: 2024-12-09T10:42:56,424 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., storeName=a9d4962ae8c180b5b7fa587448872d04/C, priority=12, startTime=1733740975957; duration=0sec 2024-12-09T10:42:56,424 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:42:56,424 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9d4962ae8c180b5b7fa587448872d04:C 2024-12-09T10:42:56,451 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/B/3d08c03a384f4177a07ba356e4dda6d6 2024-12-09T10:42:56,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/C/15c1d27748c64bbc862c3c88d63093d5 is 50, key is test_row_0/C:col10/1733740974721/Put/seqid=0 2024-12-09T10:42:56,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742315_1491 (size=12301) 2024-12-09T10:42:56,464 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/C/15c1d27748c64bbc862c3c88d63093d5 2024-12-09T10:42:56,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/A/eb72702febac4491aa661801539fbf6e as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/eb72702febac4491aa661801539fbf6e 2024-12-09T10:42:56,472 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/eb72702febac4491aa661801539fbf6e, entries=150, sequenceid=346, filesize=12.0 K 2024-12-09T10:42:56,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/B/3d08c03a384f4177a07ba356e4dda6d6 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/3d08c03a384f4177a07ba356e4dda6d6 2024-12-09T10:42:56,477 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/3d08c03a384f4177a07ba356e4dda6d6, entries=150, sequenceid=346, filesize=12.0 K 2024-12-09T10:42:56,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/C/15c1d27748c64bbc862c3c88d63093d5 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/15c1d27748c64bbc862c3c88d63093d5 2024-12-09T10:42:56,482 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/15c1d27748c64bbc862c3c88d63093d5, entries=150, sequenceid=346, filesize=12.0 K 2024-12-09T10:42:56,483 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=0 B/0 for a9d4962ae8c180b5b7fa587448872d04 in 464ms, sequenceid=346, compaction requested=false 2024-12-09T10:42:56,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for a9d4962ae8c180b5b7fa587448872d04: 2024-12-09T10:42:56,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. 2024-12-09T10:42:56,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-12-09T10:42:56,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-12-09T10:42:56,490 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-12-09T10:42:56,490 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 772 msec 2024-12-09T10:42:56,493 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 781 msec 2024-12-09T10:42:56,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-09T10:42:56,850 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T10:42:56,851 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-09T10:42:56,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees 2024-12-09T10:42:56,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-09T10:42:56,853 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T10:42:56,854 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T10:42:56,854 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T10:42:56,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(8855): Flush requested on a9d4962ae8c180b5b7fa587448872d04 2024-12-09T10:42:56,933 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a9d4962ae8c180b5b7fa587448872d04 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-09T10:42:56,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9d4962ae8c180b5b7fa587448872d04, store=A 2024-12-09T10:42:56,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:42:56,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9d4962ae8c180b5b7fa587448872d04, store=B 2024-12-09T10:42:56,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:42:56,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9d4962ae8c180b5b7fa587448872d04, store=C 2024-12-09T10:42:56,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:42:56,938 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/A/65f0d378b9a147fc96c0e65b8c90a692 is 50, key is test_row_0/A:col10/1733740976924/Put/seqid=0 2024-12-09T10:42:56,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742316_1492 (size=19621) 2024-12-09T10:42:56,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-09T10:42:57,007 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-09T10:42:57,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. 2024-12-09T10:42:57,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. as already flushing 2024-12-09T10:42:57,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. 2024-12-09T10:42:57,007 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:57,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:57,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:57,021 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733740976991, max timestamp=1733740977017Num uniques 859; 2024-12-09T10:42:57,021 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733740976991, max timestamp=1733740977017Num uniques 859; 2024-12-09T10:42:57,021 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733740976991, max timestamp=1733740977017Num uniques 859; 2024-12-09T10:42:57,021 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T10:42:57,021 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T10:42:57,021 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T10:42:57,125 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:57,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59136 deadline: 1733741037111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:57,126 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:57,126 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:57,126 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:57,127 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:57,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59156 deadline: 1733741037113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:57,127 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:57,127 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:57,127 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:57,160 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-09T10:42:57,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. 2024-12-09T10:42:57,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. as already flushing 2024-12-09T10:42:57,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. 2024-12-09T10:42:57,160 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:57,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:57,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:57,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-09T10:42:57,238 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:57,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59136 deadline: 1733741037229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:57,239 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:57,239 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:57,239 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:57,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59156 deadline: 1733741037229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:57,239 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:57,239 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:57,239 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:57,239 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:57,313 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-09T10:42:57,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. 2024-12-09T10:42:57,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. as already flushing 2024-12-09T10:42:57,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. 2024-12-09T10:42:57,314 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:57,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:57,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:57,342 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/A/65f0d378b9a147fc96c0e65b8c90a692 2024-12-09T10:42:57,349 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/B/da0482b8255e47e390a7dbce9f573057 is 50, key is test_row_0/B:col10/1733740976924/Put/seqid=0 2024-12-09T10:42:57,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742317_1493 (size=12301) 2024-12-09T10:42:57,459 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:57,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59136 deadline: 1733741037449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:57,460 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:57,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59156 deadline: 1733741037449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:57,460 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:57,460 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:57,460 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:57,460 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:57,460 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:57,460 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:57,466 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-09T10:42:57,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. 2024-12-09T10:42:57,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. as already flushing 2024-12-09T10:42:57,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. 2024-12-09T10:42:57,467 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:57,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:57,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:57,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-09T10:42:57,619 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-09T10:42:57,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. 2024-12-09T10:42:57,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. as already flushing 2024-12-09T10:42:57,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. 2024-12-09T10:42:57,620 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:57,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:57,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:57,753 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/B/da0482b8255e47e390a7dbce9f573057 2024-12-09T10:42:57,759 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/C/a338146550ce49188a0f56e8ec209400 is 50, key is test_row_0/C:col10/1733740976924/Put/seqid=0 2024-12-09T10:42:57,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742318_1494 (size=12301) 2024-12-09T10:42:57,772 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-09T10:42:57,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. 2024-12-09T10:42:57,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. as already flushing 2024-12-09T10:42:57,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. 2024-12-09T10:42:57,772 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:57,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:57,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:57,781 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:57,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59136 deadline: 1733741037769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:57,782 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:57,782 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:57,782 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:57,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59156 deadline: 1733741037769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:57,782 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:57,782 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:57,783 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:57,783 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:57,881 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:57,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59114 deadline: 1733741037869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:57,881 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:42:57,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59126 deadline: 1733741037869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:42:57,882 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 7ea43b39fc24,36101,1733740827678 for 'test_row_2' in a9d4962ae8c180b5b7fa587448872d04 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4239 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T10:42:57,882 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 7ea43b39fc24,36101,1733740827678 for 'test_row_1' in a9d4962ae8c180b5b7fa587448872d04 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4240 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T10:42:57,882 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:57,882 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:57,882 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 , the old value is region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:57,882 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:57,882 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:42:57,882 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., hostname=7ea43b39fc24,36101,1733740827678, seqNum=2 because the exception is null or not the one we care about 2024-12-09T10:42:57,925 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-09T10:42:57,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. 2024-12-09T10:42:57,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. as already flushing 2024-12-09T10:42:57,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. 2024-12-09T10:42:57,925 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:57,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:57,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:57,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-09T10:42:58,078 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-09T10:42:58,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. 2024-12-09T10:42:58,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. as already flushing 2024-12-09T10:42:58,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. 2024-12-09T10:42:58,078 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:58,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:58,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:42:58,166 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/C/a338146550ce49188a0f56e8ec209400 2024-12-09T10:42:58,171 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/A/65f0d378b9a147fc96c0e65b8c90a692 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/65f0d378b9a147fc96c0e65b8c90a692 2024-12-09T10:42:58,174 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/65f0d378b9a147fc96c0e65b8c90a692, entries=300, sequenceid=357, filesize=19.2 K 2024-12-09T10:42:58,175 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/B/da0482b8255e47e390a7dbce9f573057 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/da0482b8255e47e390a7dbce9f573057 2024-12-09T10:42:58,178 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/da0482b8255e47e390a7dbce9f573057, entries=150, sequenceid=357, filesize=12.0 K 2024-12-09T10:42:58,178 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/C/a338146550ce49188a0f56e8ec209400 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/a338146550ce49188a0f56e8ec209400 2024-12-09T10:42:58,181 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/a338146550ce49188a0f56e8ec209400, entries=150, sequenceid=357, filesize=12.0 K 2024-12-09T10:42:58,182 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for a9d4962ae8c180b5b7fa587448872d04 in 1249ms, sequenceid=357, compaction requested=true 2024-12-09T10:42:58,182 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a9d4962ae8c180b5b7fa587448872d04: 2024-12-09T10:42:58,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9d4962ae8c180b5b7fa587448872d04:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T10:42:58,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:42:58,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9d4962ae8c180b5b7fa587448872d04:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T10:42:58,182 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T10:42:58,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:42:58,182 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T10:42:58,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9d4962ae8c180b5b7fa587448872d04:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T10:42:58,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T10:42:58,183 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T10:42:58,183 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 44905 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T10:42:58,183 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1541): a9d4962ae8c180b5b7fa587448872d04/B is initiating minor compaction (all files) 2024-12-09T10:42:58,183 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1541): a9d4962ae8c180b5b7fa587448872d04/A is initiating minor compaction (all files) 2024-12-09T10:42:58,183 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a9d4962ae8c180b5b7fa587448872d04/B in TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. 2024-12-09T10:42:58,183 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a9d4962ae8c180b5b7fa587448872d04/A in TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. 2024-12-09T10:42:58,183 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/2dfb0722739a4698960871f1343a1285, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/3d08c03a384f4177a07ba356e4dda6d6, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/da0482b8255e47e390a7dbce9f573057] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp, totalSize=36.7 K 2024-12-09T10:42:58,183 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/db824540a2184f01b017efbf56bdfb90, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/eb72702febac4491aa661801539fbf6e, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/65f0d378b9a147fc96c0e65b8c90a692] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp, totalSize=43.9 K 2024-12-09T10:42:58,184 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 2dfb0722739a4698960871f1343a1285, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1733740973592 2024-12-09T10:42:58,184 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting db824540a2184f01b017efbf56bdfb90, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1733740973592 2024-12-09T10:42:58,184 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d08c03a384f4177a07ba356e4dda6d6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1733740974692 2024-12-09T10:42:58,184 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting eb72702febac4491aa661801539fbf6e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1733740974692 2024-12-09T10:42:58,184 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting da0482b8255e47e390a7dbce9f573057, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1733740976924 2024-12-09T10:42:58,185 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 65f0d378b9a147fc96c0e65b8c90a692, keycount=300, bloomtype=ROW, size=19.2 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1733740976920 2024-12-09T10:42:58,191 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a9d4962ae8c180b5b7fa587448872d04#A#compaction#422 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:42:58,191 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/A/50ffcc17b20f47a7b0dd86672351d650 is 50, key is test_row_0/A:col10/1733740976924/Put/seqid=0 2024-12-09T10:42:58,192 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a9d4962ae8c180b5b7fa587448872d04#B#compaction#423 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:42:58,193 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/B/6166035185a149bfac70e4224b87edab is 50, key is test_row_0/B:col10/1733740976924/Put/seqid=0 2024-12-09T10:42:58,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742319_1495 (size=13085) 2024-12-09T10:42:58,209 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/B/6166035185a149bfac70e4224b87edab as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/6166035185a149bfac70e4224b87edab 2024-12-09T10:42:58,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742320_1496 (size=13085) 2024-12-09T10:42:58,213 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a9d4962ae8c180b5b7fa587448872d04/B of a9d4962ae8c180b5b7fa587448872d04 into 6166035185a149bfac70e4224b87edab(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:42:58,213 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a9d4962ae8c180b5b7fa587448872d04: 2024-12-09T10:42:58,213 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., storeName=a9d4962ae8c180b5b7fa587448872d04/B, priority=13, startTime=1733740978182; duration=0sec 2024-12-09T10:42:58,213 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T10:42:58,213 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9d4962ae8c180b5b7fa587448872d04:B 2024-12-09T10:42:58,214 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T10:42:58,214 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T10:42:58,214 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1541): a9d4962ae8c180b5b7fa587448872d04/C is initiating minor compaction (all files) 2024-12-09T10:42:58,214 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a9d4962ae8c180b5b7fa587448872d04/C in TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. 2024-12-09T10:42:58,215 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/b39d155fec7d439b9bf4a265f21d46c9, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/15c1d27748c64bbc862c3c88d63093d5, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/a338146550ce49188a0f56e8ec209400] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp, totalSize=36.7 K 2024-12-09T10:42:58,215 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting b39d155fec7d439b9bf4a265f21d46c9, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1733740973592 2024-12-09T10:42:58,215 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/A/50ffcc17b20f47a7b0dd86672351d650 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/50ffcc17b20f47a7b0dd86672351d650 2024-12-09T10:42:58,215 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 15c1d27748c64bbc862c3c88d63093d5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1733740974692 2024-12-09T10:42:58,216 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting a338146550ce49188a0f56e8ec209400, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1733740976924 2024-12-09T10:42:58,222 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a9d4962ae8c180b5b7fa587448872d04/A of a9d4962ae8c180b5b7fa587448872d04 into 50ffcc17b20f47a7b0dd86672351d650(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:42:58,222 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a9d4962ae8c180b5b7fa587448872d04: 2024-12-09T10:42:58,222 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., storeName=a9d4962ae8c180b5b7fa587448872d04/A, priority=13, startTime=1733740978182; duration=0sec 2024-12-09T10:42:58,222 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:42:58,222 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9d4962ae8c180b5b7fa587448872d04:A 2024-12-09T10:42:58,226 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a9d4962ae8c180b5b7fa587448872d04#C#compaction#424 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:42:58,226 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/C/5758f57b12a24f739a18052332528472 is 50, key is test_row_0/C:col10/1733740976924/Put/seqid=0 2024-12-09T10:42:58,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742321_1497 (size=13051) 2024-12-09T10:42:58,230 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-09T10:42:58,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. 2024-12-09T10:42:58,231 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2902): Flushing a9d4962ae8c180b5b7fa587448872d04 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-12-09T10:42:58,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9d4962ae8c180b5b7fa587448872d04, store=A 2024-12-09T10:42:58,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T10:42:58,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9d4962ae8c180b5b7fa587448872d04, store=B 2024-12-09T10:42:58,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T10:42:58,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9d4962ae8c180b5b7fa587448872d04, store=C 2024-12-09T10:42:58,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T10:42:58,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/A/e725584357e44272ac2490834b32d501 is 50, key is test_row_0/A:col10/1733740977111/Put/seqid=0 2024-12-09T10:42:58,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742322_1498 (size=12301) 2024-12-09T10:42:58,238 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=396 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/A/e725584357e44272ac2490834b32d501 2024-12-09T10:42:58,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(8855): Flush requested on a9d4962ae8c180b5b7fa587448872d04 2024-12-09T10:42:58,244 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. as already flushing 2024-12-09T10:42:58,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/B/0e98d93ff7334f99aa6fd6663f73e561 is 50, key is test_row_0/B:col10/1733740977111/Put/seqid=0 2024-12-09T10:42:58,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742323_1499 (size=12301) 2024-12-09T10:42:58,250 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=396 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/B/0e98d93ff7334f99aa6fd6663f73e561 2024-12-09T10:42:58,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/C/4512225ab9294a13b8af1454a6c42fd3 is 50, key is test_row_0/C:col10/1733740977111/Put/seqid=0 2024-12-09T10:42:58,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742324_1500 (size=12301) 2024-12-09T10:42:58,281 INFO [Thread-1941 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1941. 2024-12-09T10:42:58,282 DEBUG [Thread-1941 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T10:42:58,282 DEBUG [Thread-1941 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:42:58,282 DEBUG [Thread-1941 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:42:58,282 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T10:42:58,284 INFO [Thread-1926 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1926. 2024-12-09T10:42:58,284 DEBUG [Thread-1926 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T10:42:58,284 DEBUG [Thread-1926 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:42:58,284 INFO [Thread-1943 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1943. 2024-12-09T10:42:58,284 DEBUG [Thread-1943 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T10:42:58,284 DEBUG [Thread-1943 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:42:58,284 DEBUG [Thread-1926 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:42:58,284 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T10:42:58,285 DEBUG [Thread-1943 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:42:58,285 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T10:42:58,285 INFO [Thread-1945 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1945. 2024-12-09T10:42:58,285 DEBUG [Thread-1945 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T10:42:58,285 DEBUG [Thread-1945 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:42:58,285 DEBUG [Thread-1945 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:42:58,286 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T10:42:58,291 INFO [Thread-1934 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1934. 2024-12-09T10:42:58,291 DEBUG [Thread-1934 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T10:42:58,291 DEBUG [Thread-1934 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:42:58,291 DEBUG [Thread-1934 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:42:58,291 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T10:42:58,293 INFO [Thread-1932 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1932. 2024-12-09T10:42:58,293 DEBUG [Thread-1932 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T10:42:58,293 DEBUG [Thread-1932 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:42:58,293 DEBUG [Thread-1932 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:42:58,293 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T10:42:58,294 INFO [Thread-1939 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1939. 2024-12-09T10:42:58,294 DEBUG [Thread-1939 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T10:42:58,294 DEBUG [Thread-1939 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:42:58,294 DEBUG [Thread-1939 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:42:58,294 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T10:42:58,296 INFO [Thread-1937 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1937. 2024-12-09T10:42:58,296 DEBUG [Thread-1937 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T10:42:58,296 DEBUG [Thread-1937 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:42:58,297 DEBUG [Thread-1937 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:42:58,297 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T10:42:58,634 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/C/5758f57b12a24f739a18052332528472 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/5758f57b12a24f739a18052332528472 2024-12-09T10:42:58,637 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a9d4962ae8c180b5b7fa587448872d04/C of a9d4962ae8c180b5b7fa587448872d04 into 5758f57b12a24f739a18052332528472(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:42:58,637 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a9d4962ae8c180b5b7fa587448872d04: 2024-12-09T10:42:58,637 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., storeName=a9d4962ae8c180b5b7fa587448872d04/C, priority=13, startTime=1733740978182; duration=0sec 2024-12-09T10:42:58,637 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:42:58,637 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9d4962ae8c180b5b7fa587448872d04:C 2024-12-09T10:42:58,679 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=396 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/C/4512225ab9294a13b8af1454a6c42fd3 2024-12-09T10:42:58,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/A/e725584357e44272ac2490834b32d501 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/e725584357e44272ac2490834b32d501 2024-12-09T10:42:58,685 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/e725584357e44272ac2490834b32d501, entries=150, sequenceid=396, filesize=12.0 K 2024-12-09T10:42:58,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/B/0e98d93ff7334f99aa6fd6663f73e561 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/0e98d93ff7334f99aa6fd6663f73e561 2024-12-09T10:42:58,688 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/0e98d93ff7334f99aa6fd6663f73e561, entries=150, sequenceid=396, filesize=12.0 K 2024-12-09T10:42:58,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/C/4512225ab9294a13b8af1454a6c42fd3 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/4512225ab9294a13b8af1454a6c42fd3 2024-12-09T10:42:58,691 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/4512225ab9294a13b8af1454a6c42fd3, entries=150, sequenceid=396, filesize=12.0 K 2024-12-09T10:42:58,692 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for a9d4962ae8c180b5b7fa587448872d04 in 461ms, sequenceid=396, compaction requested=false 2024-12-09T10:42:58,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2603): Flush status journal for a9d4962ae8c180b5b7fa587448872d04: 2024-12-09T10:42:58,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. 2024-12-09T10:42:58,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136 2024-12-09T10:42:58,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.HMaster(4169): Remote procedure done, pid=136 2024-12-09T10:42:58,694 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-12-09T10:42:58,694 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8390 sec 2024-12-09T10:42:58,696 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees in 1.8440 sec 2024-12-09T10:42:59,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-09T10:42:59,000 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T10:43:01,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(8855): Flush requested on a9d4962ae8c180b5b7fa587448872d04 2024-12-09T10:43:01,900 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a9d4962ae8c180b5b7fa587448872d04 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-09T10:43:01,900 INFO [Thread-1928 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1928. 2024-12-09T10:43:01,900 DEBUG [Thread-1928 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T10:43:01,900 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9d4962ae8c180b5b7fa587448872d04, store=A 2024-12-09T10:43:01,900 DEBUG [Thread-1928 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:43:01,900 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:43:01,900 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9d4962ae8c180b5b7fa587448872d04, store=B 2024-12-09T10:43:01,900 DEBUG [Thread-1928 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:43:01,900 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:43:01,900 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9d4962ae8c180b5b7fa587448872d04, store=C 2024-12-09T10:43:01,900 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:43:01,900 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T10:43:01,904 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/A/b6e899d044f0404b884658d3a9ed03c6 is 50, key is test_row_0/A:col10/1733740978290/Put/seqid=0 2024-12-09T10:43:01,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742325_1501 (size=12301) 2024-12-09T10:43:01,920 INFO [Thread-1930 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1930. 2024-12-09T10:43:01,920 DEBUG [Thread-1930 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T10:43:01,921 DEBUG [Thread-1930 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:43:01,921 DEBUG [Thread-1930 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:43:01,921 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T10:43:01,921 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-09T10:43:01,921 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 43 2024-12-09T10:43:01,921 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 54 2024-12-09T10:43:01,921 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 60 2024-12-09T10:43:01,921 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 75 2024-12-09T10:43:01,921 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 99 2024-12-09T10:43:01,921 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-09T10:43:01,921 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-09T10:43:01,921 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1124 2024-12-09T10:43:01,921 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3371 rows 2024-12-09T10:43:01,921 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1124 2024-12-09T10:43:01,921 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3372 rows 2024-12-09T10:43:01,921 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1122 2024-12-09T10:43:01,921 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3365 rows 2024-12-09T10:43:01,921 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1122 2024-12-09T10:43:01,921 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3366 rows 2024-12-09T10:43:01,921 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1125 2024-12-09T10:43:01,921 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3373 rows 2024-12-09T10:43:01,921 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-09T10:43:01,921 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T10:43:01,921 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:43:01,921 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:43:01,921 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T10:43:01,922 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-12-09T10:43:01,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-09T10:43:01,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-09T10:43:01,925 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733740981925"}]},"ts":"1733740981925"} 2024-12-09T10:43:01,926 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-09T10:43:01,926 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-09T10:43:01,927 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-09T10:43:01,928 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a9d4962ae8c180b5b7fa587448872d04, UNASSIGN}] 2024-12-09T10:43:01,928 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a9d4962ae8c180b5b7fa587448872d04, UNASSIGN 2024-12-09T10:43:01,929 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=a9d4962ae8c180b5b7fa587448872d04, regionState=CLOSING, regionLocation=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:01,930 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a9d4962ae8c180b5b7fa587448872d04, UNASSIGN because future has completed 2024-12-09T10:43:01,931 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-09T10:43:01,931 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; CloseRegionProcedure a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678}] 2024-12-09T10:43:02,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-09T10:43:02,083 INFO [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(122): Close a9d4962ae8c180b5b7fa587448872d04 2024-12-09T10:43:02,083 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-09T10:43:02,083 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1722): Closing a9d4962ae8c180b5b7fa587448872d04, disabling compactions & flushes 2024-12-09T10:43:02,083 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. 2024-12-09T10:43:02,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-09T10:43:02,308 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=408 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/A/b6e899d044f0404b884658d3a9ed03c6 2024-12-09T10:43:02,314 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/B/f8d5b622b0294e2fa5cd7c8f43e3a0d3 is 50, key is test_row_0/B:col10/1733740978290/Put/seqid=0 2024-12-09T10:43:02,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742326_1502 (size=12301) 2024-12-09T10:43:02,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-09T10:43:02,717 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=408 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/B/f8d5b622b0294e2fa5cd7c8f43e3a0d3 2024-12-09T10:43:02,724 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/C/9c1d0717263845d18156a9c24cdc17e7 is 50, key is test_row_0/C:col10/1733740978290/Put/seqid=0 2024-12-09T10:43:02,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742327_1503 (size=12301) 2024-12-09T10:43:03,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-09T10:43:03,127 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=408 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/C/9c1d0717263845d18156a9c24cdc17e7 2024-12-09T10:43:03,131 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/A/b6e899d044f0404b884658d3a9ed03c6 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/b6e899d044f0404b884658d3a9ed03c6 2024-12-09T10:43:03,133 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/b6e899d044f0404b884658d3a9ed03c6, entries=150, sequenceid=408, filesize=12.0 K 2024-12-09T10:43:03,134 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/B/f8d5b622b0294e2fa5cd7c8f43e3a0d3 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/f8d5b622b0294e2fa5cd7c8f43e3a0d3 2024-12-09T10:43:03,136 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/f8d5b622b0294e2fa5cd7c8f43e3a0d3, entries=150, sequenceid=408, filesize=12.0 K 2024-12-09T10:43:03,137 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/C/9c1d0717263845d18156a9c24cdc17e7 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/9c1d0717263845d18156a9c24cdc17e7 2024-12-09T10:43:03,139 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/9c1d0717263845d18156a9c24cdc17e7, entries=150, sequenceid=408, filesize=12.0 K 2024-12-09T10:43:03,140 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=6.71 KB/6870 for a9d4962ae8c180b5b7fa587448872d04 in 1240ms, sequenceid=408, compaction requested=true 2024-12-09T10:43:03,140 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a9d4962ae8c180b5b7fa587448872d04: 2024-12-09T10:43:03,140 INFO [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. 2024-12-09T10:43:03,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9d4962ae8c180b5b7fa587448872d04:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T10:43:03,140 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. 2024-12-09T10:43:03,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:43:03,140 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. after waiting 0 ms 2024-12-09T10:43:03,140 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. because compaction request was cancelled 2024-12-09T10:43:03,140 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. 2024-12-09T10:43:03,140 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9d4962ae8c180b5b7fa587448872d04:A 2024-12-09T10:43:03,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9d4962ae8c180b5b7fa587448872d04:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T10:43:03,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:43:03,140 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. because compaction request was cancelled 2024-12-09T10:43:03,140 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9d4962ae8c180b5b7fa587448872d04:B 2024-12-09T10:43:03,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9d4962ae8c180b5b7fa587448872d04:C, priority=-2147483648, current under compaction store size is 2 2024-12-09T10:43:03,141 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. because compaction request was cancelled 2024-12-09T10:43:03,141 INFO [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(2902): Flushing a9d4962ae8c180b5b7fa587448872d04 3/3 column families, dataSize=6.71 KB heapSize=20.67 KB 2024-12-09T10:43:03,141 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9d4962ae8c180b5b7fa587448872d04:C 2024-12-09T10:43:03,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:43:03,141 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9d4962ae8c180b5b7fa587448872d04, store=A 2024-12-09T10:43:03,141 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:43:03,141 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9d4962ae8c180b5b7fa587448872d04, store=B 2024-12-09T10:43:03,141 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:43:03,141 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9d4962ae8c180b5b7fa587448872d04, store=C 2024-12-09T10:43:03,141 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:43:03,144 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/A/d997df81d461490380cb719f19ed9a4c is 50, key is test_row_1/A:col10/1733740981919/Put/seqid=0 2024-12-09T10:43:03,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742328_1504 (size=7415) 2024-12-09T10:43:03,547 INFO [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=412 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/A/d997df81d461490380cb719f19ed9a4c 2024-12-09T10:43:03,553 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/B/e5d2c1d5617f44c7a8897ae3f11fc7a0 is 50, key is test_row_1/B:col10/1733740981919/Put/seqid=0 2024-12-09T10:43:03,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742329_1505 (size=7415) 2024-12-09T10:43:03,956 INFO [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=412 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/B/e5d2c1d5617f44c7a8897ae3f11fc7a0 2024-12-09T10:43:03,961 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/C/fe56ed98c044468d8f5b53559bcb1939 is 50, key is test_row_1/C:col10/1733740981919/Put/seqid=0 2024-12-09T10:43:03,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742330_1506 (size=7415) 2024-12-09T10:43:04,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-09T10:43:04,365 INFO [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=412 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/C/fe56ed98c044468d8f5b53559bcb1939 2024-12-09T10:43:04,368 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/A/d997df81d461490380cb719f19ed9a4c as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/d997df81d461490380cb719f19ed9a4c 2024-12-09T10:43:04,371 INFO [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/d997df81d461490380cb719f19ed9a4c, entries=50, sequenceid=412, filesize=7.2 K 2024-12-09T10:43:04,372 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/B/e5d2c1d5617f44c7a8897ae3f11fc7a0 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/e5d2c1d5617f44c7a8897ae3f11fc7a0 2024-12-09T10:43:04,374 INFO [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/e5d2c1d5617f44c7a8897ae3f11fc7a0, entries=50, sequenceid=412, filesize=7.2 K 2024-12-09T10:43:04,375 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/.tmp/C/fe56ed98c044468d8f5b53559bcb1939 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/fe56ed98c044468d8f5b53559bcb1939 2024-12-09T10:43:04,378 INFO [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/fe56ed98c044468d8f5b53559bcb1939, entries=50, sequenceid=412, filesize=7.2 K 2024-12-09T10:43:04,378 INFO [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(3140): Finished flush of dataSize ~6.71 KB/6870, heapSize ~20.63 KB/21120, currentSize=0 B/0 for a9d4962ae8c180b5b7fa587448872d04 in 1238ms, sequenceid=412, compaction requested=true 2024-12-09T10:43:04,379 DEBUG [StoreCloser-TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/ede6cf7684cf4dc7ba6efe0144fb3879, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/8885e4f83c1d48f2b3f5a3764b7624bb, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/a810fcf87a914544963f68a74ef348af, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/63b80a85f93a420497405757144f06e9, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/0d982244b1bd4c1d8f8991e834f152e8, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/a77c95defe88421e9cc93b097e36a1e4, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/e78d092557904c17beda2f7b0aea08b6, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/3fe0357bead24995ab0df9001b0a758a, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/6eebef8d6f96424db6d866097fd1c8c6, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/d6986f3396e84565b5cc5755f71cda4c, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/9bb012ad13804bf7a0596b186ed951af, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/58304220a6644c908c5a8c77acbe05d8, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/512c59480a7f49ee92e3b82a36856686, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/dd96c1ed5f474d1f9d890acd3e6c5bc3, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/43c06e50d8ec4d1aaf6070bf893b7331, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/166732208ef842fa83e58177ff49cbed, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/c7729feaaba04e168cd44303728f3c7c, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/761a2ab0f27f4743be169602f9bfe798, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/1633ce36d61f4331ba8f98e701fade04, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/eff44505efda40b2bc47227d510dc082, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/db824540a2184f01b017efbf56bdfb90, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/eb72702febac4491aa661801539fbf6e, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/65f0d378b9a147fc96c0e65b8c90a692] to archive 2024-12-09T10:43:04,380 DEBUG [StoreCloser-TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T10:43:04,381 DEBUG [StoreCloser-TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/ede6cf7684cf4dc7ba6efe0144fb3879 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/ede6cf7684cf4dc7ba6efe0144fb3879 2024-12-09T10:43:04,382 DEBUG [StoreCloser-TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/8885e4f83c1d48f2b3f5a3764b7624bb to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/8885e4f83c1d48f2b3f5a3764b7624bb 2024-12-09T10:43:04,383 DEBUG [StoreCloser-TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/a810fcf87a914544963f68a74ef348af to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/a810fcf87a914544963f68a74ef348af 2024-12-09T10:43:04,383 DEBUG [StoreCloser-TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/63b80a85f93a420497405757144f06e9 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/63b80a85f93a420497405757144f06e9 2024-12-09T10:43:04,384 DEBUG [StoreCloser-TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/0d982244b1bd4c1d8f8991e834f152e8 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/0d982244b1bd4c1d8f8991e834f152e8 2024-12-09T10:43:04,385 DEBUG [StoreCloser-TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/a77c95defe88421e9cc93b097e36a1e4 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/a77c95defe88421e9cc93b097e36a1e4 2024-12-09T10:43:04,386 DEBUG [StoreCloser-TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/e78d092557904c17beda2f7b0aea08b6 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/e78d092557904c17beda2f7b0aea08b6 2024-12-09T10:43:04,387 DEBUG [StoreCloser-TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/3fe0357bead24995ab0df9001b0a758a to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/3fe0357bead24995ab0df9001b0a758a 2024-12-09T10:43:04,388 DEBUG [StoreCloser-TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/6eebef8d6f96424db6d866097fd1c8c6 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/6eebef8d6f96424db6d866097fd1c8c6 2024-12-09T10:43:04,388 DEBUG [StoreCloser-TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/d6986f3396e84565b5cc5755f71cda4c to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/d6986f3396e84565b5cc5755f71cda4c 2024-12-09T10:43:04,389 DEBUG [StoreCloser-TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/9bb012ad13804bf7a0596b186ed951af to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/9bb012ad13804bf7a0596b186ed951af 2024-12-09T10:43:04,390 DEBUG [StoreCloser-TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/58304220a6644c908c5a8c77acbe05d8 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/58304220a6644c908c5a8c77acbe05d8 2024-12-09T10:43:04,390 DEBUG [StoreCloser-TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/512c59480a7f49ee92e3b82a36856686 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/512c59480a7f49ee92e3b82a36856686 2024-12-09T10:43:04,391 DEBUG [StoreCloser-TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/dd96c1ed5f474d1f9d890acd3e6c5bc3 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/dd96c1ed5f474d1f9d890acd3e6c5bc3 2024-12-09T10:43:04,392 DEBUG [StoreCloser-TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/43c06e50d8ec4d1aaf6070bf893b7331 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/43c06e50d8ec4d1aaf6070bf893b7331 2024-12-09T10:43:04,393 DEBUG [StoreCloser-TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/166732208ef842fa83e58177ff49cbed to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/166732208ef842fa83e58177ff49cbed 2024-12-09T10:43:04,393 DEBUG [StoreCloser-TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/c7729feaaba04e168cd44303728f3c7c to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/c7729feaaba04e168cd44303728f3c7c 2024-12-09T10:43:04,394 DEBUG [StoreCloser-TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/761a2ab0f27f4743be169602f9bfe798 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/761a2ab0f27f4743be169602f9bfe798 2024-12-09T10:43:04,395 DEBUG [StoreCloser-TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/1633ce36d61f4331ba8f98e701fade04 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/1633ce36d61f4331ba8f98e701fade04 2024-12-09T10:43:04,396 DEBUG [StoreCloser-TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/eff44505efda40b2bc47227d510dc082 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/eff44505efda40b2bc47227d510dc082 2024-12-09T10:43:04,396 DEBUG [StoreCloser-TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/db824540a2184f01b017efbf56bdfb90 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/db824540a2184f01b017efbf56bdfb90 2024-12-09T10:43:04,397 DEBUG [StoreCloser-TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/eb72702febac4491aa661801539fbf6e to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/eb72702febac4491aa661801539fbf6e 2024-12-09T10:43:04,398 DEBUG [StoreCloser-TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/65f0d378b9a147fc96c0e65b8c90a692 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/65f0d378b9a147fc96c0e65b8c90a692 2024-12-09T10:43:04,399 DEBUG [StoreCloser-TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/91643f4180224083bde6bc6e1e0761e7, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/147c57e1cfce4d39a46151273fff679e, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/7c7bf0b746e941258bfb587778e6797e, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/082a49cd746b411783e06a0445e2c82a, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/5078744e03c24fcaa89b373b49cb40ed, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/824f3705cc1f49528a8254d1439c06b0, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/7621085d7d964420a2c0f71470975b91, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/815471661efe40d29a8b77b7552b6c3c, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/27fdc4df16ac436cb9d58bef806c5b09, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/0590e692e7154225a6985b40d5133ae8, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/d73bb15157a146258389d3e6e09218bd, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/3615777c8aed4c0da3ffbf1b270c4a90, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/24f011f67790402e942e1cb09ee3b373, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/a781c4afec684d37bf0a336c88e9d515, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/677fcc713ff14dee891a99f14b93868d, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/6a0e0a4648504a47bd647515526435ad, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/88d2d65910ce43e4adee06171ab91f10, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/b1077b97e6c849ec938ca2979362a2db, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/b0529cfaae2d4ecb918091868205ac55, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/2dfb0722739a4698960871f1343a1285, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/a520bba8420c40489c93573e721767a2, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/3d08c03a384f4177a07ba356e4dda6d6, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/da0482b8255e47e390a7dbce9f573057] to archive 2024-12-09T10:43:04,400 DEBUG [StoreCloser-TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T10:43:04,401 DEBUG [StoreCloser-TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/91643f4180224083bde6bc6e1e0761e7 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/91643f4180224083bde6bc6e1e0761e7 2024-12-09T10:43:04,402 DEBUG [StoreCloser-TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/147c57e1cfce4d39a46151273fff679e to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/147c57e1cfce4d39a46151273fff679e 2024-12-09T10:43:04,402 DEBUG [StoreCloser-TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/7c7bf0b746e941258bfb587778e6797e to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/7c7bf0b746e941258bfb587778e6797e 2024-12-09T10:43:04,403 DEBUG [StoreCloser-TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/082a49cd746b411783e06a0445e2c82a to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/082a49cd746b411783e06a0445e2c82a 2024-12-09T10:43:04,404 DEBUG [StoreCloser-TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/5078744e03c24fcaa89b373b49cb40ed to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/5078744e03c24fcaa89b373b49cb40ed 2024-12-09T10:43:04,405 DEBUG [StoreCloser-TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/824f3705cc1f49528a8254d1439c06b0 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/824f3705cc1f49528a8254d1439c06b0 2024-12-09T10:43:04,406 DEBUG [StoreCloser-TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/7621085d7d964420a2c0f71470975b91 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/7621085d7d964420a2c0f71470975b91 2024-12-09T10:43:04,406 DEBUG [StoreCloser-TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/815471661efe40d29a8b77b7552b6c3c to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/815471661efe40d29a8b77b7552b6c3c 2024-12-09T10:43:04,407 DEBUG [StoreCloser-TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/27fdc4df16ac436cb9d58bef806c5b09 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/27fdc4df16ac436cb9d58bef806c5b09 2024-12-09T10:43:04,408 DEBUG [StoreCloser-TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/0590e692e7154225a6985b40d5133ae8 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/0590e692e7154225a6985b40d5133ae8 2024-12-09T10:43:04,409 DEBUG [StoreCloser-TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/d73bb15157a146258389d3e6e09218bd to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/d73bb15157a146258389d3e6e09218bd 2024-12-09T10:43:04,410 DEBUG [StoreCloser-TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/3615777c8aed4c0da3ffbf1b270c4a90 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/3615777c8aed4c0da3ffbf1b270c4a90 2024-12-09T10:43:04,410 DEBUG [StoreCloser-TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/24f011f67790402e942e1cb09ee3b373 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/24f011f67790402e942e1cb09ee3b373 2024-12-09T10:43:04,411 DEBUG [StoreCloser-TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/a781c4afec684d37bf0a336c88e9d515 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/a781c4afec684d37bf0a336c88e9d515 2024-12-09T10:43:04,412 DEBUG [StoreCloser-TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/677fcc713ff14dee891a99f14b93868d to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/677fcc713ff14dee891a99f14b93868d 2024-12-09T10:43:04,413 DEBUG [StoreCloser-TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/6a0e0a4648504a47bd647515526435ad to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/6a0e0a4648504a47bd647515526435ad 2024-12-09T10:43:04,414 DEBUG [StoreCloser-TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/88d2d65910ce43e4adee06171ab91f10 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/88d2d65910ce43e4adee06171ab91f10 2024-12-09T10:43:04,414 DEBUG [StoreCloser-TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/b1077b97e6c849ec938ca2979362a2db to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/b1077b97e6c849ec938ca2979362a2db 2024-12-09T10:43:04,415 DEBUG [StoreCloser-TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/b0529cfaae2d4ecb918091868205ac55 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/b0529cfaae2d4ecb918091868205ac55 2024-12-09T10:43:04,416 DEBUG [StoreCloser-TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/2dfb0722739a4698960871f1343a1285 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/2dfb0722739a4698960871f1343a1285 2024-12-09T10:43:04,416 DEBUG [StoreCloser-TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/a520bba8420c40489c93573e721767a2 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/a520bba8420c40489c93573e721767a2 2024-12-09T10:43:04,417 DEBUG [StoreCloser-TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/3d08c03a384f4177a07ba356e4dda6d6 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/3d08c03a384f4177a07ba356e4dda6d6 2024-12-09T10:43:04,418 DEBUG [StoreCloser-TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/da0482b8255e47e390a7dbce9f573057 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/da0482b8255e47e390a7dbce9f573057 2024-12-09T10:43:04,419 DEBUG [StoreCloser-TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/fc075148dd294d26a23184328dee613f, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/8c74baf9f4cf489da0805f3115b9de5f, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/a4078266a2174e2ebab4f3417a0dc153, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/92a4e0df517f43ad90a32b359c891932, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/e07da91f4679436ea54c2fa4102feb57, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/3781b717cc594df3958a69e047ec4176, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/2d2fc7f4b82b425e9af977c87f18319d, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/c1c5efacdb2f46b9bfec1e38e4c0fc6e, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/7021dd8209f246229cad52ca59f31fe9, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/fe66cb496d904e0da0d60997990f353b, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/56f1394847c84e6eb3ba5f5be953e616, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/9dc3c216f0264622824688e3f1366272, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/9fb6f147b0ad44d493a159ebd2c70d92, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/2269591a633c4010baf8548207af79b0, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/71084c55670644f3afa26a3a496798ed, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/27058d4d5b7d4e53b364c81a0ed2ab42, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/7f0074694757496c8a31bdfcd7bfb026, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/d87cc844c4ab4030aa8f1985aa864598, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/b39d155fec7d439b9bf4a265f21d46c9, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/0c8aa6099be54bcf898a31e41012af66, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/15c1d27748c64bbc862c3c88d63093d5, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/a338146550ce49188a0f56e8ec209400] to archive 2024-12-09T10:43:04,420 DEBUG [StoreCloser-TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T10:43:04,421 DEBUG [StoreCloser-TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/fc075148dd294d26a23184328dee613f to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/fc075148dd294d26a23184328dee613f 2024-12-09T10:43:04,421 DEBUG [StoreCloser-TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/8c74baf9f4cf489da0805f3115b9de5f to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/8c74baf9f4cf489da0805f3115b9de5f 2024-12-09T10:43:04,422 DEBUG [StoreCloser-TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/a4078266a2174e2ebab4f3417a0dc153 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/a4078266a2174e2ebab4f3417a0dc153 2024-12-09T10:43:04,423 DEBUG [StoreCloser-TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/92a4e0df517f43ad90a32b359c891932 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/92a4e0df517f43ad90a32b359c891932 2024-12-09T10:43:04,424 DEBUG [StoreCloser-TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/e07da91f4679436ea54c2fa4102feb57 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/e07da91f4679436ea54c2fa4102feb57 2024-12-09T10:43:04,425 DEBUG [StoreCloser-TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/3781b717cc594df3958a69e047ec4176 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/3781b717cc594df3958a69e047ec4176 2024-12-09T10:43:04,425 DEBUG [StoreCloser-TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/2d2fc7f4b82b425e9af977c87f18319d to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/2d2fc7f4b82b425e9af977c87f18319d 2024-12-09T10:43:04,426 DEBUG [StoreCloser-TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/c1c5efacdb2f46b9bfec1e38e4c0fc6e to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/c1c5efacdb2f46b9bfec1e38e4c0fc6e 2024-12-09T10:43:04,427 DEBUG [StoreCloser-TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/7021dd8209f246229cad52ca59f31fe9 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/7021dd8209f246229cad52ca59f31fe9 2024-12-09T10:43:04,428 DEBUG [StoreCloser-TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/fe66cb496d904e0da0d60997990f353b to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/fe66cb496d904e0da0d60997990f353b 2024-12-09T10:43:04,428 DEBUG [StoreCloser-TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/56f1394847c84e6eb3ba5f5be953e616 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/56f1394847c84e6eb3ba5f5be953e616 2024-12-09T10:43:04,429 DEBUG [StoreCloser-TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/9dc3c216f0264622824688e3f1366272 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/9dc3c216f0264622824688e3f1366272 2024-12-09T10:43:04,430 DEBUG [StoreCloser-TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/9fb6f147b0ad44d493a159ebd2c70d92 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/9fb6f147b0ad44d493a159ebd2c70d92 2024-12-09T10:43:04,430 DEBUG [StoreCloser-TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/2269591a633c4010baf8548207af79b0 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/2269591a633c4010baf8548207af79b0 2024-12-09T10:43:04,431 DEBUG [StoreCloser-TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/71084c55670644f3afa26a3a496798ed to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/71084c55670644f3afa26a3a496798ed 2024-12-09T10:43:04,432 DEBUG [StoreCloser-TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/27058d4d5b7d4e53b364c81a0ed2ab42 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/27058d4d5b7d4e53b364c81a0ed2ab42 2024-12-09T10:43:04,433 DEBUG [StoreCloser-TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/7f0074694757496c8a31bdfcd7bfb026 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/7f0074694757496c8a31bdfcd7bfb026 2024-12-09T10:43:04,433 DEBUG [StoreCloser-TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/d87cc844c4ab4030aa8f1985aa864598 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/d87cc844c4ab4030aa8f1985aa864598 2024-12-09T10:43:04,434 DEBUG [StoreCloser-TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/b39d155fec7d439b9bf4a265f21d46c9 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/b39d155fec7d439b9bf4a265f21d46c9 2024-12-09T10:43:04,435 DEBUG [StoreCloser-TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/0c8aa6099be54bcf898a31e41012af66 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/0c8aa6099be54bcf898a31e41012af66 2024-12-09T10:43:04,435 DEBUG [StoreCloser-TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/15c1d27748c64bbc862c3c88d63093d5 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/15c1d27748c64bbc862c3c88d63093d5 2024-12-09T10:43:04,436 DEBUG [StoreCloser-TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/a338146550ce49188a0f56e8ec209400 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/a338146550ce49188a0f56e8ec209400 2024-12-09T10:43:04,440 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/recovered.edits/415.seqid, newMaxSeqId=415, maxSeqId=1 2024-12-09T10:43:04,440 INFO [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. 2024-12-09T10:43:04,441 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1676): Region close journal for a9d4962ae8c180b5b7fa587448872d04: Waiting for close lock at 1733740982083Running coprocessor pre-close hooks at 1733740982083Disabling compacts and flushes for region at 1733740982083Disabling writes for close at 1733740983140 (+1057 ms)Obtaining lock to block concurrent updates at 1733740983141 (+1 ms)Preparing flush snapshotting stores in a9d4962ae8c180b5b7fa587448872d04 at 1733740983141Finished memstore snapshotting TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04., syncing WAL and waiting on mvcc, flushsize=dataSize=6870, getHeapSize=21120, getOffHeapSize=0, getCellsCount=150 at 1733740983141Flushing stores of TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04. at 1733740983141Flushing a9d4962ae8c180b5b7fa587448872d04/A: creating writer at 1733740983141Flushing a9d4962ae8c180b5b7fa587448872d04/A: appending metadata at 1733740983143 (+2 ms)Flushing a9d4962ae8c180b5b7fa587448872d04/A: closing flushed file at 1733740983143Flushing a9d4962ae8c180b5b7fa587448872d04/B: creating writer at 1733740983550 (+407 ms)Flushing a9d4962ae8c180b5b7fa587448872d04/B: appending metadata at 1733740983552 (+2 ms)Flushing a9d4962ae8c180b5b7fa587448872d04/B: closing flushed file at 1733740983552Flushing a9d4962ae8c180b5b7fa587448872d04/C: creating writer at 1733740983959 (+407 ms)Flushing a9d4962ae8c180b5b7fa587448872d04/C: appending metadata at 1733740983961 (+2 ms)Flushing a9d4962ae8c180b5b7fa587448872d04/C: closing flushed file at 1733740983961Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5e1a5bcd: reopening flushed file at 1733740984368 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7199bf72: reopening flushed file at 1733740984371 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5b0a6e8c: reopening flushed file at 1733740984375 (+4 ms)Finished flush of dataSize ~6.71 KB/6870, heapSize ~20.63 KB/21120, currentSize=0 B/0 for a9d4962ae8c180b5b7fa587448872d04 in 1238ms, sequenceid=412, compaction requested=true at 1733740984378 (+3 ms)Writing region close event to WAL at 1733740984437 (+59 ms)Running coprocessor post-close hooks at 1733740984440 (+3 ms)Closed at 1733740984440 2024-12-09T10:43:04,442 INFO [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(157): Closed a9d4962ae8c180b5b7fa587448872d04 2024-12-09T10:43:04,443 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=a9d4962ae8c180b5b7fa587448872d04, regionState=CLOSED 2024-12-09T10:43:04,444 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=140, ppid=139, state=RUNNABLE, hasLock=false; CloseRegionProcedure a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 because future has completed 2024-12-09T10:43:04,446 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-12-09T10:43:04,446 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; CloseRegionProcedure a9d4962ae8c180b5b7fa587448872d04, server=7ea43b39fc24,36101,1733740827678 in 2.5140 sec 2024-12-09T10:43:04,447 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=139, resume processing ppid=138 2024-12-09T10:43:04,447 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, ppid=138, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a9d4962ae8c180b5b7fa587448872d04, UNASSIGN in 2.5180 sec 2024-12-09T10:43:04,449 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-12-09T10:43:04,449 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.5210 sec 2024-12-09T10:43:04,450 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733740984450"}]},"ts":"1733740984450"} 2024-12-09T10:43:04,451 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-09T10:43:04,451 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-09T10:43:04,453 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.5300 sec 2024-12-09T10:43:06,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-09T10:43:06,090 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-09T10:43:06,091 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-12-09T10:43:06,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] procedure2.ProcedureExecutor(1139): Stored pid=141, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T10:43:06,092 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=141, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T10:43:06,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-12-09T10:43:06,093 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=141, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T10:43:06,095 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04 2024-12-09T10:43:06,096 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A, FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B, FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C, FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/recovered.edits] 2024-12-09T10:43:06,099 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/50ffcc17b20f47a7b0dd86672351d650 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/50ffcc17b20f47a7b0dd86672351d650 2024-12-09T10:43:06,100 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/b6e899d044f0404b884658d3a9ed03c6 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/b6e899d044f0404b884658d3a9ed03c6 2024-12-09T10:43:06,101 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/d997df81d461490380cb719f19ed9a4c to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/d997df81d461490380cb719f19ed9a4c 2024-12-09T10:43:06,102 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/e725584357e44272ac2490834b32d501 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/A/e725584357e44272ac2490834b32d501 2024-12-09T10:43:06,104 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/0e98d93ff7334f99aa6fd6663f73e561 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/0e98d93ff7334f99aa6fd6663f73e561 2024-12-09T10:43:06,104 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/6166035185a149bfac70e4224b87edab to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/6166035185a149bfac70e4224b87edab 2024-12-09T10:43:06,105 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/e5d2c1d5617f44c7a8897ae3f11fc7a0 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/e5d2c1d5617f44c7a8897ae3f11fc7a0 2024-12-09T10:43:06,106 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/f8d5b622b0294e2fa5cd7c8f43e3a0d3 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/B/f8d5b622b0294e2fa5cd7c8f43e3a0d3 2024-12-09T10:43:06,108 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/4512225ab9294a13b8af1454a6c42fd3 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/4512225ab9294a13b8af1454a6c42fd3 2024-12-09T10:43:06,109 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/5758f57b12a24f739a18052332528472 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/5758f57b12a24f739a18052332528472 2024-12-09T10:43:06,109 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/9c1d0717263845d18156a9c24cdc17e7 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/9c1d0717263845d18156a9c24cdc17e7 2024-12-09T10:43:06,110 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/fe56ed98c044468d8f5b53559bcb1939 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/C/fe56ed98c044468d8f5b53559bcb1939 2024-12-09T10:43:06,112 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/recovered.edits/415.seqid to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04/recovered.edits/415.seqid 2024-12-09T10:43:06,113 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/a9d4962ae8c180b5b7fa587448872d04 2024-12-09T10:43:06,113 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-09T10:43:06,114 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=141, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T10:43:06,116 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-09T10:43:06,118 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-09T10:43:06,119 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=141, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T10:43:06,119 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-09T10:43:06,119 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733740986119"}]},"ts":"9223372036854775807"} 2024-12-09T10:43:06,121 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-09T10:43:06,121 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => a9d4962ae8c180b5b7fa587448872d04, NAME => 'TestAcidGuarantees,,1733740956079.a9d4962ae8c180b5b7fa587448872d04.', STARTKEY => '', ENDKEY => ''}] 2024-12-09T10:43:06,121 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-09T10:43:06,121 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733740986121"}]},"ts":"9223372036854775807"} 2024-12-09T10:43:06,123 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-09T10:43:06,124 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=141, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T10:43:06,124 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 33 msec 2024-12-09T10:43:06,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-12-09T10:43:06,200 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-09T10:43:06,200 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-09T10:43:06,210 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=248 (was 252), OpenFileDescriptor=445 (was 459), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=317 (was 351), ProcessCount=11 (was 11), AvailableMemoryMB=7302 (was 7361) 2024-12-09T10:43:06,220 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=248, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=317, ProcessCount=11, AvailableMemoryMB=7301 2024-12-09T10:43:06,221 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-09T10:43:06,221 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T10:43:06,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] procedure2.ProcedureExecutor(1139): Stored pid=142, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-09T10:43:06,223 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-09T10:43:06,223 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:06,223 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 142 2024-12-09T10:43:06,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-12-09T10:43:06,224 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-09T10:43:06,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742331_1507 (size=1061) 2024-12-09T10:43:06,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-12-09T10:43:06,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-12-09T10:43:06,630 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160 2024-12-09T10:43:06,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742332_1508 (size=53) 2024-12-09T10:43:06,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-12-09T10:43:07,035 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T10:43:07,035 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 92c0b293dd887d05ae6f705c47b9c951, disabling compactions & flushes 2024-12-09T10:43:07,035 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:07,035 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:07,035 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. after waiting 0 ms 2024-12-09T10:43:07,035 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:07,035 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:07,035 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 92c0b293dd887d05ae6f705c47b9c951: Waiting for close lock at 1733740987035Disabling compacts and flushes for region at 1733740987035Disabling writes for close at 1733740987035Writing region close event to WAL at 1733740987035Closed at 1733740987035 2024-12-09T10:43:07,036 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-09T10:43:07,037 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733740987037"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733740987037"}]},"ts":"1733740987037"} 2024-12-09T10:43:07,038 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-09T10:43:07,039 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-09T10:43:07,039 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733740987039"}]},"ts":"1733740987039"} 2024-12-09T10:43:07,040 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-09T10:43:07,040 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=92c0b293dd887d05ae6f705c47b9c951, ASSIGN}] 2024-12-09T10:43:07,041 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=92c0b293dd887d05ae6f705c47b9c951, ASSIGN 2024-12-09T10:43:07,042 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=92c0b293dd887d05ae6f705c47b9c951, ASSIGN; state=OFFLINE, location=7ea43b39fc24,36101,1733740827678; forceNewPlan=false, retain=false 2024-12-09T10:43:07,193 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=92c0b293dd887d05ae6f705c47b9c951, regionState=OPENING, regionLocation=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:07,194 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=92c0b293dd887d05ae6f705c47b9c951, ASSIGN because future has completed 2024-12-09T10:43:07,194 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE, hasLock=false; OpenRegionProcedure 92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678}] 2024-12-09T10:43:07,343 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-09T10:43:07,349 INFO [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:07,349 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(7752): Opening region: {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} 2024-12-09T10:43:07,349 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 92c0b293dd887d05ae6f705c47b9c951 2024-12-09T10:43:07,349 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T10:43:07,349 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(7794): checking encryption for 92c0b293dd887d05ae6f705c47b9c951 2024-12-09T10:43:07,349 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(7797): checking classloading for 92c0b293dd887d05ae6f705c47b9c951 2024-12-09T10:43:07,350 INFO [StoreOpener-92c0b293dd887d05ae6f705c47b9c951-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 92c0b293dd887d05ae6f705c47b9c951 2024-12-09T10:43:07,351 INFO [StoreOpener-92c0b293dd887d05ae6f705c47b9c951-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T10:43:07,352 INFO [StoreOpener-92c0b293dd887d05ae6f705c47b9c951-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 92c0b293dd887d05ae6f705c47b9c951 columnFamilyName A 2024-12-09T10:43:07,352 DEBUG [StoreOpener-92c0b293dd887d05ae6f705c47b9c951-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:07,352 INFO [StoreOpener-92c0b293dd887d05ae6f705c47b9c951-1 {}] regionserver.HStore(327): Store=92c0b293dd887d05ae6f705c47b9c951/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T10:43:07,352 INFO [StoreOpener-92c0b293dd887d05ae6f705c47b9c951-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 92c0b293dd887d05ae6f705c47b9c951 2024-12-09T10:43:07,353 INFO [StoreOpener-92c0b293dd887d05ae6f705c47b9c951-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T10:43:07,353 INFO [StoreOpener-92c0b293dd887d05ae6f705c47b9c951-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 92c0b293dd887d05ae6f705c47b9c951 columnFamilyName B 2024-12-09T10:43:07,353 DEBUG [StoreOpener-92c0b293dd887d05ae6f705c47b9c951-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:07,353 INFO [StoreOpener-92c0b293dd887d05ae6f705c47b9c951-1 {}] regionserver.HStore(327): Store=92c0b293dd887d05ae6f705c47b9c951/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T10:43:07,354 INFO [StoreOpener-92c0b293dd887d05ae6f705c47b9c951-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 92c0b293dd887d05ae6f705c47b9c951 2024-12-09T10:43:07,354 INFO [StoreOpener-92c0b293dd887d05ae6f705c47b9c951-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T10:43:07,354 INFO [StoreOpener-92c0b293dd887d05ae6f705c47b9c951-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 92c0b293dd887d05ae6f705c47b9c951 columnFamilyName C 2024-12-09T10:43:07,354 DEBUG [StoreOpener-92c0b293dd887d05ae6f705c47b9c951-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:07,355 INFO [StoreOpener-92c0b293dd887d05ae6f705c47b9c951-1 {}] regionserver.HStore(327): Store=92c0b293dd887d05ae6f705c47b9c951/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T10:43:07,355 INFO [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:07,355 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1038): replaying wal for 92c0b293dd887d05ae6f705c47b9c951 2024-12-09T10:43:07,356 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951 2024-12-09T10:43:07,356 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951 2024-12-09T10:43:07,356 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1048): stopping wal replay for 92c0b293dd887d05ae6f705c47b9c951 2024-12-09T10:43:07,357 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1060): Cleaning up temporary data for 92c0b293dd887d05ae6f705c47b9c951 2024-12-09T10:43:07,357 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-09T10:43:07,358 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1093): writing seq id for 92c0b293dd887d05ae6f705c47b9c951 2024-12-09T10:43:07,359 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T10:43:07,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-12-09T10:43:07,360 INFO [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1114): Opened 92c0b293dd887d05ae6f705c47b9c951; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62609265, jitterRate=-0.06704925000667572}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-09T10:43:07,360 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 92c0b293dd887d05ae6f705c47b9c951 2024-12-09T10:43:07,360 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1006): Region open journal for 92c0b293dd887d05ae6f705c47b9c951: Running coprocessor pre-open hook at 1733740987349Writing region info on filesystem at 1733740987349Initializing all the Stores at 1733740987350 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733740987350Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733740987350Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733740987350Cleaning up temporary data from old regions at 1733740987357 (+7 ms)Running coprocessor post-open hooks at 1733740987360 (+3 ms)Region opened successfully at 1733740987360 2024-12-09T10:43:07,361 INFO [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., pid=144, masterSystemTime=1733740987346 2024-12-09T10:43:07,362 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:07,363 INFO [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:07,363 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=92c0b293dd887d05ae6f705c47b9c951, regionState=OPEN, openSeqNum=2, regionLocation=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:07,364 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=144, ppid=143, state=RUNNABLE, hasLock=false; OpenRegionProcedure 92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 because future has completed 2024-12-09T10:43:07,366 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-12-09T10:43:07,366 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; OpenRegionProcedure 92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 in 171 msec 2024-12-09T10:43:07,368 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=142 2024-12-09T10:43:07,368 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=142, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=92c0b293dd887d05ae6f705c47b9c951, ASSIGN in 326 msec 2024-12-09T10:43:07,368 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-09T10:43:07,368 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733740987368"}]},"ts":"1733740987368"} 2024-12-09T10:43:07,369 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-09T10:43:07,370 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-09T10:43:07,372 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1490 sec 2024-12-09T10:43:08,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-12-09T10:43:08,370 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-09T10:43:08,371 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28949a5a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:43:08,371 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ea43b39fc24,35095,-1 for getting cluster id 2024-12-09T10:43:08,371 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T10:43:08,372 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0945b5ae-b46b-4b8f-af48-56b0bb92df09' 2024-12-09T10:43:08,372 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T10:43:08,373 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0945b5ae-b46b-4b8f-af48-56b0bb92df09" 2024-12-09T10:43:08,373 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1db4f580, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:43:08,373 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ea43b39fc24,35095,-1] 2024-12-09T10:43:08,373 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T10:43:08,373 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:43:08,374 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55848, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T10:43:08,374 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12212b47, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:43:08,375 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T10:43:08,375 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ea43b39fc24,36101,1733740827678, seqNum=-1] 2024-12-09T10:43:08,375 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T10:43:08,376 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33412, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T10:43:08,377 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-09T10:43:08,377 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 7ea43b39fc24,35095,1733740826918 2024-12-09T10:43:08,377 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@50c5e0dd 2024-12-09T10:43:08,378 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-09T10:43:08,378 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55850, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-09T10:43:08,379 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-09T10:43:08,380 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T10:43:08,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] procedure2.ProcedureExecutor(1139): Stored pid=145, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-09T10:43:08,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742333_1509 (size=1097) 2024-12-09T10:43:08,390 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-09T10:43:08,390 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-09T10:43:08,391 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-09T10:43:08,393 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=92c0b293dd887d05ae6f705c47b9c951, REOPEN/MOVE}] 2024-12-09T10:43:08,393 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=92c0b293dd887d05ae6f705c47b9c951, REOPEN/MOVE 2024-12-09T10:43:08,394 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=92c0b293dd887d05ae6f705c47b9c951, regionState=CLOSING, regionLocation=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:08,395 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=92c0b293dd887d05ae6f705c47b9c951, REOPEN/MOVE because future has completed 2024-12-09T10:43:08,395 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-09T10:43:08,396 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=148, ppid=147, state=RUNNABLE, hasLock=false; CloseRegionProcedure 92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678}] 2024-12-09T10:43:08,547 INFO [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] handler.UnassignRegionHandler(122): Close 92c0b293dd887d05ae6f705c47b9c951 2024-12-09T10:43:08,547 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-09T10:43:08,547 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1722): Closing 92c0b293dd887d05ae6f705c47b9c951, disabling compactions & flushes 2024-12-09T10:43:08,548 INFO [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:08,548 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:08,548 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. after waiting 0 ms 2024-12-09T10:43:08,548 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:08,551 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-09T10:43:08,551 INFO [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:08,551 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1676): Region close journal for 92c0b293dd887d05ae6f705c47b9c951: Waiting for close lock at 1733740988547Running coprocessor pre-close hooks at 1733740988547Disabling compacts and flushes for region at 1733740988547Disabling writes for close at 1733740988548 (+1 ms)Writing region close event to WAL at 1733740988548Running coprocessor post-close hooks at 1733740988551 (+3 ms)Closed at 1733740988551 2024-12-09T10:43:08,551 WARN [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegionServer(3299): Not adding moved region record: 92c0b293dd887d05ae6f705c47b9c951 to self. 2024-12-09T10:43:08,553 INFO [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] handler.UnassignRegionHandler(157): Closed 92c0b293dd887d05ae6f705c47b9c951 2024-12-09T10:43:08,553 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=92c0b293dd887d05ae6f705c47b9c951, regionState=CLOSED 2024-12-09T10:43:08,555 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=148, ppid=147, state=RUNNABLE, hasLock=false; CloseRegionProcedure 92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 because future has completed 2024-12-09T10:43:08,557 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=148, resume processing ppid=147 2024-12-09T10:43:08,557 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, ppid=147, state=SUCCESS, hasLock=false; CloseRegionProcedure 92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 in 160 msec 2024-12-09T10:43:08,557 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=92c0b293dd887d05ae6f705c47b9c951, REOPEN/MOVE; state=CLOSED, location=7ea43b39fc24,36101,1733740827678; forceNewPlan=false, retain=true 2024-12-09T10:43:08,708 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=92c0b293dd887d05ae6f705c47b9c951, regionState=OPENING, regionLocation=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:08,710 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=92c0b293dd887d05ae6f705c47b9c951, REOPEN/MOVE because future has completed 2024-12-09T10:43:08,710 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=147, state=RUNNABLE, hasLock=false; OpenRegionProcedure 92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678}] 2024-12-09T10:43:08,864 INFO [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:08,864 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(7752): Opening region: {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} 2024-12-09T10:43:08,864 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 92c0b293dd887d05ae6f705c47b9c951 2024-12-09T10:43:08,864 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T10:43:08,865 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(7794): checking encryption for 92c0b293dd887d05ae6f705c47b9c951 2024-12-09T10:43:08,865 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(7797): checking classloading for 92c0b293dd887d05ae6f705c47b9c951 2024-12-09T10:43:08,866 INFO [StoreOpener-92c0b293dd887d05ae6f705c47b9c951-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 92c0b293dd887d05ae6f705c47b9c951 2024-12-09T10:43:08,866 INFO [StoreOpener-92c0b293dd887d05ae6f705c47b9c951-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T10:43:08,866 INFO [StoreOpener-92c0b293dd887d05ae6f705c47b9c951-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 92c0b293dd887d05ae6f705c47b9c951 columnFamilyName A 2024-12-09T10:43:08,867 DEBUG [StoreOpener-92c0b293dd887d05ae6f705c47b9c951-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:08,868 INFO [StoreOpener-92c0b293dd887d05ae6f705c47b9c951-1 {}] regionserver.HStore(327): Store=92c0b293dd887d05ae6f705c47b9c951/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T10:43:08,868 INFO [StoreOpener-92c0b293dd887d05ae6f705c47b9c951-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 92c0b293dd887d05ae6f705c47b9c951 2024-12-09T10:43:08,868 INFO [StoreOpener-92c0b293dd887d05ae6f705c47b9c951-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T10:43:08,868 INFO [StoreOpener-92c0b293dd887d05ae6f705c47b9c951-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 92c0b293dd887d05ae6f705c47b9c951 columnFamilyName B 2024-12-09T10:43:08,868 DEBUG [StoreOpener-92c0b293dd887d05ae6f705c47b9c951-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:08,869 INFO [StoreOpener-92c0b293dd887d05ae6f705c47b9c951-1 {}] regionserver.HStore(327): Store=92c0b293dd887d05ae6f705c47b9c951/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T10:43:08,869 INFO [StoreOpener-92c0b293dd887d05ae6f705c47b9c951-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 92c0b293dd887d05ae6f705c47b9c951 2024-12-09T10:43:08,869 INFO [StoreOpener-92c0b293dd887d05ae6f705c47b9c951-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T10:43:08,869 INFO [StoreOpener-92c0b293dd887d05ae6f705c47b9c951-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 92c0b293dd887d05ae6f705c47b9c951 columnFamilyName C 2024-12-09T10:43:08,869 DEBUG [StoreOpener-92c0b293dd887d05ae6f705c47b9c951-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:08,869 INFO [StoreOpener-92c0b293dd887d05ae6f705c47b9c951-1 {}] regionserver.HStore(327): Store=92c0b293dd887d05ae6f705c47b9c951/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T10:43:08,870 INFO [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:08,870 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1038): replaying wal for 92c0b293dd887d05ae6f705c47b9c951 2024-12-09T10:43:08,870 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951 2024-12-09T10:43:08,871 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951 2024-12-09T10:43:08,872 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1048): stopping wal replay for 92c0b293dd887d05ae6f705c47b9c951 2024-12-09T10:43:08,872 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1060): Cleaning up temporary data for 92c0b293dd887d05ae6f705c47b9c951 2024-12-09T10:43:08,872 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-09T10:43:08,873 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1093): writing seq id for 92c0b293dd887d05ae6f705c47b9c951 2024-12-09T10:43:08,874 INFO [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1114): Opened 92c0b293dd887d05ae6f705c47b9c951; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68988902, jitterRate=0.02801474928855896}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-09T10:43:08,874 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 92c0b293dd887d05ae6f705c47b9c951 2024-12-09T10:43:08,874 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1006): Region open journal for 92c0b293dd887d05ae6f705c47b9c951: Running coprocessor pre-open hook at 1733740988865Writing region info on filesystem at 1733740988865Initializing all the Stores at 1733740988865Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733740988865Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733740988865Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733740988865Cleaning up temporary data from old regions at 1733740988872 (+7 ms)Running coprocessor post-open hooks at 1733740988874 (+2 ms)Region opened successfully at 1733740988874 2024-12-09T10:43:08,875 INFO [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., pid=149, masterSystemTime=1733740988861 2024-12-09T10:43:08,877 DEBUG [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:08,877 INFO [RS_OPEN_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:08,877 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=92c0b293dd887d05ae6f705c47b9c951, regionState=OPEN, openSeqNum=5, regionLocation=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:08,878 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=149, ppid=147, state=RUNNABLE, hasLock=false; OpenRegionProcedure 92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 because future has completed 2024-12-09T10:43:08,880 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=147 2024-12-09T10:43:08,880 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=147, state=SUCCESS, hasLock=false; OpenRegionProcedure 92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 in 169 msec 2024-12-09T10:43:08,882 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-12-09T10:43:08,882 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=92c0b293dd887d05ae6f705c47b9c951, REOPEN/MOVE in 487 msec 2024-12-09T10:43:08,884 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=146, resume processing ppid=145 2024-12-09T10:43:08,884 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, ppid=145, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 491 msec 2024-12-09T10:43:08,885 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 504 msec 2024-12-09T10:43:08,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-12-09T10:43:08,886 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-09T10:43:08,887 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@755fd6d8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:43:08,887 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ea43b39fc24,35095,-1 for getting cluster id 2024-12-09T10:43:08,887 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T10:43:08,888 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0945b5ae-b46b-4b8f-af48-56b0bb92df09' 2024-12-09T10:43:08,888 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T10:43:08,888 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0945b5ae-b46b-4b8f-af48-56b0bb92df09" 2024-12-09T10:43:08,888 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f5288b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:43:08,888 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ea43b39fc24,35095,-1] 2024-12-09T10:43:08,888 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T10:43:08,888 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:43:08,889 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35814, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T10:43:08,890 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b5a742d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:43:08,890 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@367c3300, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:43:08,890 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ea43b39fc24,35095,-1 for getting cluster id 2024-12-09T10:43:08,890 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T10:43:08,891 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0945b5ae-b46b-4b8f-af48-56b0bb92df09' 2024-12-09T10:43:08,891 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T10:43:08,891 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0945b5ae-b46b-4b8f-af48-56b0bb92df09" 2024-12-09T10:43:08,891 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@192faab3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:43:08,891 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ea43b39fc24,35095,-1] 2024-12-09T10:43:08,891 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T10:43:08,891 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:43:08,892 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35840, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T10:43:08,892 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c2a537d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:43:08,893 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@699e95cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:43:08,893 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ea43b39fc24,35095,-1 for getting cluster id 2024-12-09T10:43:08,893 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T10:43:08,893 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0945b5ae-b46b-4b8f-af48-56b0bb92df09' 2024-12-09T10:43:08,894 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T10:43:08,894 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0945b5ae-b46b-4b8f-af48-56b0bb92df09" 2024-12-09T10:43:08,894 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44cd1ef6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:43:08,894 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ea43b39fc24,35095,-1] 2024-12-09T10:43:08,894 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T10:43:08,894 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:43:08,895 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35862, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T10:43:08,895 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@368121b7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:43:08,896 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@228c97bb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:43:08,896 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ea43b39fc24,35095,-1 for getting cluster id 2024-12-09T10:43:08,896 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T10:43:08,896 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0945b5ae-b46b-4b8f-af48-56b0bb92df09' 2024-12-09T10:43:08,896 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T10:43:08,896 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0945b5ae-b46b-4b8f-af48-56b0bb92df09" 2024-12-09T10:43:08,897 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5358045b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:43:08,897 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ea43b39fc24,35095,-1] 2024-12-09T10:43:08,897 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T10:43:08,897 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:43:08,897 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35886, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T10:43:08,898 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7776c3b5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:43:08,898 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1aee49bf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:43:08,898 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ea43b39fc24,35095,-1 for getting cluster id 2024-12-09T10:43:08,898 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T10:43:08,899 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0945b5ae-b46b-4b8f-af48-56b0bb92df09' 2024-12-09T10:43:08,899 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T10:43:08,899 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0945b5ae-b46b-4b8f-af48-56b0bb92df09" 2024-12-09T10:43:08,899 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30359b06, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:43:08,899 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ea43b39fc24,35095,-1] 2024-12-09T10:43:08,899 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T10:43:08,899 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:43:08,900 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35906, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T10:43:08,901 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7215e3c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:43:08,901 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f9ceb79, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:43:08,901 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ea43b39fc24,35095,-1 for getting cluster id 2024-12-09T10:43:08,901 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T10:43:08,902 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0945b5ae-b46b-4b8f-af48-56b0bb92df09' 2024-12-09T10:43:08,902 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T10:43:08,902 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0945b5ae-b46b-4b8f-af48-56b0bb92df09" 2024-12-09T10:43:08,902 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f38c330, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:43:08,902 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ea43b39fc24,35095,-1] 2024-12-09T10:43:08,902 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T10:43:08,902 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:43:08,903 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35922, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T10:43:08,903 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7bd4bcad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:43:08,904 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3aa9a0d6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:43:08,904 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ea43b39fc24,35095,-1 for getting cluster id 2024-12-09T10:43:08,904 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T10:43:08,904 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0945b5ae-b46b-4b8f-af48-56b0bb92df09' 2024-12-09T10:43:08,905 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T10:43:08,905 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0945b5ae-b46b-4b8f-af48-56b0bb92df09" 2024-12-09T10:43:08,905 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2103660d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:43:08,905 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ea43b39fc24,35095,-1] 2024-12-09T10:43:08,905 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T10:43:08,905 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:43:08,906 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35936, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T10:43:08,906 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ec67cc3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:43:08,907 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31d332ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:43:08,907 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ea43b39fc24,35095,-1 for getting cluster id 2024-12-09T10:43:08,907 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T10:43:08,907 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0945b5ae-b46b-4b8f-af48-56b0bb92df09' 2024-12-09T10:43:08,907 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T10:43:08,907 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0945b5ae-b46b-4b8f-af48-56b0bb92df09" 2024-12-09T10:43:08,908 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@647e38d9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:43:08,908 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ea43b39fc24,35095,-1] 2024-12-09T10:43:08,908 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T10:43:08,908 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:43:08,909 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35950, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T10:43:08,909 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70b32257, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:43:08,910 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24493301, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:43:08,910 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ea43b39fc24,35095,-1 for getting cluster id 2024-12-09T10:43:08,910 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T10:43:08,910 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0945b5ae-b46b-4b8f-af48-56b0bb92df09' 2024-12-09T10:43:08,910 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T10:43:08,910 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0945b5ae-b46b-4b8f-af48-56b0bb92df09" 2024-12-09T10:43:08,910 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b65aea1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:43:08,910 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ea43b39fc24,35095,-1] 2024-12-09T10:43:08,911 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T10:43:08,911 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:43:08,911 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35962, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T10:43:08,912 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42f702c6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:43:08,912 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f3fbbeb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:43:08,912 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ea43b39fc24,35095,-1 for getting cluster id 2024-12-09T10:43:08,912 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T10:43:08,913 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0945b5ae-b46b-4b8f-af48-56b0bb92df09' 2024-12-09T10:43:08,913 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T10:43:08,913 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0945b5ae-b46b-4b8f-af48-56b0bb92df09" 2024-12-09T10:43:08,913 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15ba7df5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:43:08,913 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ea43b39fc24,35095,-1] 2024-12-09T10:43:08,913 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T10:43:08,913 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:43:08,914 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35972, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T10:43:08,914 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@629c7ce9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T10:43:08,915 DEBUG [Thread-2287 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T10:43:08,915 DEBUG [Thread-2298 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T10:43:08,915 DEBUG [Thread-2306 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T10:43:08,915 DEBUG [Thread-2300 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T10:43:08,915 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ea43b39fc24,36101,1733740827678, seqNum=-1] 2024-12-09T10:43:08,916 DEBUG [Thread-2289 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T10:43:08,916 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ea43b39fc24,36101,1733740827678, seqNum=-1] 2024-12-09T10:43:08,916 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ea43b39fc24,36101,1733740827678, seqNum=-1] 2024-12-09T10:43:08,916 DEBUG [Thread-2291 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T10:43:08,916 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T10:43:08,916 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T10:43:08,916 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T10:43:08,916 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ea43b39fc24,36101,1733740827678, seqNum=-1] 2024-12-09T10:43:08,916 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ea43b39fc24,36101,1733740827678, seqNum=-1] 2024-12-09T10:43:08,916 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T10:43:08,917 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60464, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T10:43:08,917 DEBUG [Thread-2293 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T10:43:08,917 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T10:43:08,917 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ea43b39fc24,36101,1733740827678, seqNum=-1] 2024-12-09T10:43:08,917 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60482, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T10:43:08,917 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60466, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T10:43:08,917 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T10:43:08,918 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60494, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T10:43:08,918 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ea43b39fc24,36101,1733740827678, seqNum=-1] 2024-12-09T10:43:08,918 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-09T10:43:08,918 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60502, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T10:43:08,918 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T10:43:08,918 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60514, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T10:43:08,919 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60518, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T10:43:08,919 DEBUG [Thread-2295 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T10:43:08,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees 2024-12-09T10:43:08,920 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5] 2024-12-09T10:43:08,920 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5] 2024-12-09T10:43:08,920 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ea43b39fc24,36101,1733740827678, seqNum=-1] 2024-12-09T10:43:08,920 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T10:43:08,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-09T10:43:08,921 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5] 2024-12-09T10:43:08,921 DEBUG [Thread-2304 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T10:43:08,921 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60526, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T10:43:08,921 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T10:43:08,921 DEBUG [Thread-2302 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T10:43:08,921 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ea43b39fc24,36101,1733740827678, seqNum=-1] 2024-12-09T10:43:08,922 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T10:43:08,922 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5] 2024-12-09T10:43:08,922 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60532, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T10:43:08,923 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T10:43:08,923 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T10:43:08,923 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ea43b39fc24,36101,1733740827678, seqNum=-1] 2024-12-09T10:43:08,923 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5] 2024-12-09T10:43:08,923 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T10:43:08,925 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60540, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T10:43:08,927 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5] 2024-12-09T10:43:08,928 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5] 2024-12-09T10:43:08,929 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5] 2024-12-09T10:43:08,930 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5] 2024-12-09T10:43:08,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(8855): Flush requested on 92c0b293dd887d05ae6f705c47b9c951 2024-12-09T10:43:08,931 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5] 2024-12-09T10:43:08,939 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 92c0b293dd887d05ae6f705c47b9c951 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-09T10:43:08,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 92c0b293dd887d05ae6f705c47b9c951, store=A 2024-12-09T10:43:08,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:43:08,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 92c0b293dd887d05ae6f705c47b9c951, store=B 2024-12-09T10:43:08,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:43:08,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 92c0b293dd887d05ae6f705c47b9c951, store=C 2024-12-09T10:43:08,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:43:08,956 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:08,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60526 deadline: 1733741048950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:08,956 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:08,956 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:08,956 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:08,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60518 deadline: 1733741048953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:08,956 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:08,957 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:08,957 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:08,957 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:08,961 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:08,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60494 deadline: 1733741048955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:08,962 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:08,962 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:08,962 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:08,962 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:08,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60464 deadline: 1733741048956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:08,962 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:08,962 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:08,962 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:08,962 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:08,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60502 deadline: 1733741048960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:08,963 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:08,963 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:08,963 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:08,973 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209959ed986e3f240ba8eb0eb5ed8fe08f2_92c0b293dd887d05ae6f705c47b9c951 is 50, key is test_row_0/A:col10/1733740988934/Put/seqid=0 2024-12-09T10:43:08,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742334_1510 (size=12154) 2024-12-09T10:43:08,990 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:08,995 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209959ed986e3f240ba8eb0eb5ed8fe08f2_92c0b293dd887d05ae6f705c47b9c951 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209959ed986e3f240ba8eb0eb5ed8fe08f2_92c0b293dd887d05ae6f705c47b9c951 2024-12-09T10:43:08,996 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/A/dc7da36494e34fb1921749692776ea6f, store: [table=TestAcidGuarantees family=A region=92c0b293dd887d05ae6f705c47b9c951] 2024-12-09T10:43:08,997 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/A/dc7da36494e34fb1921749692776ea6f is 175, key is test_row_0/A:col10/1733740988934/Put/seqid=0 2024-12-09T10:43:09,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742335_1511 (size=30955) 2024-12-09T10:43:09,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-09T10:43:09,062 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:09,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60526 deadline: 1733741049060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:09,062 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:09,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60518 deadline: 1733741049060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:09,063 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:09,063 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:09,063 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:09,063 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:09,063 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:09,063 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:09,070 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:09,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60494 deadline: 1733741049069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:09,071 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:09,071 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:09,071 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:09,071 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:09,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60502 deadline: 1733741049069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:09,071 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:09,071 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:09,071 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:09,071 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:09,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60464 deadline: 1733741049069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:09,071 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:09,071 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:09,072 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:09,077 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-09T10:43:09,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:09,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. as already flushing 2024-12-09T10:43:09,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:09,077 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:43:09,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:43:09,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:43:09,230 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-09T10:43:09,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:09,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. as already flushing 2024-12-09T10:43:09,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:09,230 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:43:09,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:43:09,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:43:09,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-09T10:43:09,272 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:09,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60526 deadline: 1733741049269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:09,272 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:09,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60518 deadline: 1733741049269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:09,272 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:09,273 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:09,273 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:09,273 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:09,273 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:09,273 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:09,282 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:09,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60494 deadline: 1733741049279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:09,282 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:09,282 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:09,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60502 deadline: 1733741049280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:09,282 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:09,282 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:09,282 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:09,282 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:09,282 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:09,282 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:09,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60464 deadline: 1733741049280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:09,283 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:09,283 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:09,283 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:09,383 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-09T10:43:09,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:09,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. as already flushing 2024-12-09T10:43:09,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:09,383 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:43:09,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:43:09,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:43:09,404 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=22, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/A/dc7da36494e34fb1921749692776ea6f 2024-12-09T10:43:09,428 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/B/e252a9a06c3c47608ddaa35c36e798bc is 50, key is test_row_0/B:col10/1733740988934/Put/seqid=0 2024-12-09T10:43:09,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742336_1512 (size=12001) 2024-12-09T10:43:09,536 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-09T10:43:09,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:09,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. as already flushing 2024-12-09T10:43:09,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:09,536 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:43:09,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:43:09,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:43:09,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-09T10:43:09,581 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:09,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60526 deadline: 1733741049579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:09,581 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:09,581 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:09,581 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:09,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60518 deadline: 1733741049579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:09,581 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:09,582 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:09,582 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:09,582 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:09,592 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:09,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60494 deadline: 1733741049590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:09,592 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:09,592 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:09,592 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:09,592 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:09,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60502 deadline: 1733741049590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:09,592 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:09,592 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:09,593 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:09,593 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:09,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60464 deadline: 1733741049590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:09,593 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:09,593 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:09,593 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:09,689 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-09T10:43:09,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:09,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. as already flushing 2024-12-09T10:43:09,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:09,690 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:43:09,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:43:09,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:43:09,841 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-09T10:43:09,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:09,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. as already flushing 2024-12-09T10:43:09,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:09,842 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:43:09,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:43:09,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:43:09,853 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=22 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/B/e252a9a06c3c47608ddaa35c36e798bc 2024-12-09T10:43:09,874 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/C/3a9d00eb495849c59b9168e4d88183d2 is 50, key is test_row_0/C:col10/1733740988934/Put/seqid=0 2024-12-09T10:43:09,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742337_1513 (size=12001) 2024-12-09T10:43:09,958 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-09T10:43:09,994 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-09T10:43:09,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:09,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. as already flushing 2024-12-09T10:43:09,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:09,995 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:43:09,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:43:09,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:43:10,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-09T10:43:10,090 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:10,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60526 deadline: 1733741050089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:10,091 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:10,091 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:10,091 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:10,091 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:10,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60518 deadline: 1733741050089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:10,091 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:10,091 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:10,091 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:10,102 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:10,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60494 deadline: 1733741050099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:10,102 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:10,102 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:10,102 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:10,103 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:10,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60502 deadline: 1733741050099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:10,103 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:10,103 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:10,103 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:10,103 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:10,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60464 deadline: 1733741050099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:10,104 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:10,104 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:10,104 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:10,147 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-09T10:43:10,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:10,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. as already flushing 2024-12-09T10:43:10,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:10,148 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:43:10,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:43:10,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:43:10,281 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=22 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/C/3a9d00eb495849c59b9168e4d88183d2 2024-12-09T10:43:10,285 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/A/dc7da36494e34fb1921749692776ea6f as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/dc7da36494e34fb1921749692776ea6f 2024-12-09T10:43:10,288 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/dc7da36494e34fb1921749692776ea6f, entries=150, sequenceid=22, filesize=30.2 K 2024-12-09T10:43:10,288 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/B/e252a9a06c3c47608ddaa35c36e798bc as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/e252a9a06c3c47608ddaa35c36e798bc 2024-12-09T10:43:10,292 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/e252a9a06c3c47608ddaa35c36e798bc, entries=150, sequenceid=22, filesize=11.7 K 2024-12-09T10:43:10,292 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/C/3a9d00eb495849c59b9168e4d88183d2 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/3a9d00eb495849c59b9168e4d88183d2 2024-12-09T10:43:10,295 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/3a9d00eb495849c59b9168e4d88183d2, entries=150, sequenceid=22, filesize=11.7 K 2024-12-09T10:43:10,296 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 92c0b293dd887d05ae6f705c47b9c951 in 1358ms, sequenceid=22, compaction requested=false 2024-12-09T10:43:10,296 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-09T10:43:10,297 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 92c0b293dd887d05ae6f705c47b9c951: 2024-12-09T10:43:10,299 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-09T10:43:10,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:10,300 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2902): Flushing 92c0b293dd887d05ae6f705c47b9c951 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-09T10:43:10,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 92c0b293dd887d05ae6f705c47b9c951, store=A 2024-12-09T10:43:10,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:43:10,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 92c0b293dd887d05ae6f705c47b9c951, store=B 2024-12-09T10:43:10,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:43:10,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 92c0b293dd887d05ae6f705c47b9c951, store=C 2024-12-09T10:43:10,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:43:10,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209f6fa4996e7e745e093b59e8ba959e5de_92c0b293dd887d05ae6f705c47b9c951 is 50, key is test_row_0/A:col10/1733740988954/Put/seqid=0 2024-12-09T10:43:10,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742338_1514 (size=12154) 2024-12-09T10:43:10,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:10,715 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209f6fa4996e7e745e093b59e8ba959e5de_92c0b293dd887d05ae6f705c47b9c951 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209f6fa4996e7e745e093b59e8ba959e5de_92c0b293dd887d05ae6f705c47b9c951 2024-12-09T10:43:10,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/A/23b8ba829cf343628caa96e45fe9d3d2, store: [table=TestAcidGuarantees family=A region=92c0b293dd887d05ae6f705c47b9c951] 2024-12-09T10:43:10,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/A/23b8ba829cf343628caa96e45fe9d3d2 is 175, key is test_row_0/A:col10/1733740988954/Put/seqid=0 2024-12-09T10:43:10,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742339_1515 (size=30955) 2024-12-09T10:43:11,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-09T10:43:11,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(8855): Flush requested on 92c0b293dd887d05ae6f705c47b9c951 2024-12-09T10:43:11,103 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. as already flushing 2024-12-09T10:43:11,120 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=36, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/A/23b8ba829cf343628caa96e45fe9d3d2 2024-12-09T10:43:11,122 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:11,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60518 deadline: 1733741051118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:11,123 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:11,123 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:11,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60494 deadline: 1733741051119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:11,123 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:11,123 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:11,123 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:11,123 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:11,123 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:11,126 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:11,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60502 deadline: 1733741051121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:11,126 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:11,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60464 deadline: 1733741051121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:11,127 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:11,127 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:11,127 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:11,127 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:11,127 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:11,127 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:11,127 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:11,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60526 deadline: 1733741051122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:11,127 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:11,127 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:11,127 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:11,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/B/ce55eca3e6e640cb9d810de214302669 is 50, key is test_row_0/B:col10/1733740988954/Put/seqid=0 2024-12-09T10:43:11,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742340_1516 (size=12001) 2024-12-09T10:43:11,234 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:11,234 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:11,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60494 deadline: 1733741051230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:11,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60518 deadline: 1733741051230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:11,234 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:11,234 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:11,234 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:11,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60502 deadline: 1733741051230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:11,234 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:11,234 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:11,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60464 deadline: 1733741051230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:11,234 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:11,234 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:11,234 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:11,234 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:11,234 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:11,234 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:11,234 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:11,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60526 deadline: 1733741051230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:11,234 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:11,234 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:11,234 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:11,235 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:11,235 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:11,235 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:11,442 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:11,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60494 deadline: 1733741051440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:11,442 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:11,442 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:11,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60518 deadline: 1733741051440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:11,443 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:11,443 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:11,443 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:11,443 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:11,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60464 deadline: 1733741051440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:11,443 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:11,443 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:11,443 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:11,443 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:11,443 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:11,443 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:11,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60502 deadline: 1733741051440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:11,443 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:11,443 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:11,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60526 deadline: 1733741051440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:11,443 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:11,443 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:11,444 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:11,444 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:11,444 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:11,532 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/B/ce55eca3e6e640cb9d810de214302669 2024-12-09T10:43:11,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/C/b5d7a7ca62f34f16908ba630a7886775 is 50, key is test_row_0/C:col10/1733740988954/Put/seqid=0 2024-12-09T10:43:11,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742341_1517 (size=12001) 2024-12-09T10:43:11,752 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:11,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60494 deadline: 1733741051750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:11,753 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:11,753 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:11,753 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:11,753 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:11,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60518 deadline: 1733741051750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:11,754 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:11,754 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:11,754 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:11,754 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:11,754 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:11,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60464 deadline: 1733741051750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:11,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60502 deadline: 1733741051750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:11,754 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:11,754 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:11,754 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:11,754 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:11,754 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:11,754 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:11,754 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:11,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60526 deadline: 1733741051750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:11,754 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:11,754 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:11,754 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:11,942 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/C/b5d7a7ca62f34f16908ba630a7886775 2024-12-09T10:43:11,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/A/23b8ba829cf343628caa96e45fe9d3d2 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/23b8ba829cf343628caa96e45fe9d3d2 2024-12-09T10:43:11,950 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/23b8ba829cf343628caa96e45fe9d3d2, entries=150, sequenceid=36, filesize=30.2 K 2024-12-09T10:43:11,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/B/ce55eca3e6e640cb9d810de214302669 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/ce55eca3e6e640cb9d810de214302669 2024-12-09T10:43:11,954 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/ce55eca3e6e640cb9d810de214302669, entries=150, sequenceid=36, filesize=11.7 K 2024-12-09T10:43:11,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/C/b5d7a7ca62f34f16908ba630a7886775 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/b5d7a7ca62f34f16908ba630a7886775 2024-12-09T10:43:11,959 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/b5d7a7ca62f34f16908ba630a7886775, entries=150, sequenceid=36, filesize=11.7 K 2024-12-09T10:43:11,960 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 92c0b293dd887d05ae6f705c47b9c951 in 1660ms, sequenceid=36, compaction requested=false 2024-12-09T10:43:11,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2603): Flush status journal for 92c0b293dd887d05ae6f705c47b9c951: 2024-12-09T10:43:11,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:11,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=151 2024-12-09T10:43:11,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4169): Remote procedure done, pid=151 2024-12-09T10:43:11,963 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-12-09T10:43:11,963 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0390 sec 2024-12-09T10:43:11,965 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees in 3.0460 sec 2024-12-09T10:43:12,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(8855): Flush requested on 92c0b293dd887d05ae6f705c47b9c951 2024-12-09T10:43:12,263 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 92c0b293dd887d05ae6f705c47b9c951 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-09T10:43:12,263 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 92c0b293dd887d05ae6f705c47b9c951, store=A 2024-12-09T10:43:12,263 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:43:12,263 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 92c0b293dd887d05ae6f705c47b9c951, store=B 2024-12-09T10:43:12,263 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:43:12,263 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 92c0b293dd887d05ae6f705c47b9c951, store=C 2024-12-09T10:43:12,263 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:43:12,270 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412095d3d6025ae48437795564a3ffcc1f306_92c0b293dd887d05ae6f705c47b9c951 is 50, key is test_row_0/A:col10/1733740992261/Put/seqid=0 2024-12-09T10:43:12,284 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:12,284 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:12,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60464 deadline: 1733741052276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:12,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60502 deadline: 1733741052276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:12,284 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:12,284 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:12,284 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:12,285 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:12,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60518 deadline: 1733741052277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:12,285 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:12,285 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:12,285 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:12,285 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:12,285 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:12,285 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:12,290 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:12,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60526 deadline: 1733741052284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:12,290 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:12,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60494 deadline: 1733741052284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:12,290 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:12,290 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:12,290 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:12,291 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:12,291 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:12,291 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:12,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742342_1518 (size=12154) 2024-12-09T10:43:12,390 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:12,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60464 deadline: 1733741052389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:12,390 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:12,390 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:12,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60502 deadline: 1733741052389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:12,390 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:12,390 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:12,390 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:12,390 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:12,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60518 deadline: 1733741052389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:12,391 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:12,391 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:12,391 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:12,391 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:12,391 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:12,401 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:12,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60526 deadline: 1733741052400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:12,401 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:12,401 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:12,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60494 deadline: 1733741052400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:12,401 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:12,401 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:12,401 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:12,402 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:12,402 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:12,600 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:12,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60464 deadline: 1733741052599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:12,601 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:12,601 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:12,601 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:12,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60502 deadline: 1733741052599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:12,601 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:12,601 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:12,601 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:12,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60518 deadline: 1733741052599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:12,601 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:12,601 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:12,601 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:12,601 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:12,601 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:12,612 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:12,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60526 deadline: 1733741052610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:12,613 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:12,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60494 deadline: 1733741052610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:12,613 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:12,613 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:12,613 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:12,613 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:12,613 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:12,613 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:12,695 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:12,699 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412095d3d6025ae48437795564a3ffcc1f306_92c0b293dd887d05ae6f705c47b9c951 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412095d3d6025ae48437795564a3ffcc1f306_92c0b293dd887d05ae6f705c47b9c951 2024-12-09T10:43:12,700 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/A/adc7d04859af409a8392ab390d0db49c, store: [table=TestAcidGuarantees family=A region=92c0b293dd887d05ae6f705c47b9c951] 2024-12-09T10:43:12,700 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/A/adc7d04859af409a8392ab390d0db49c is 175, key is test_row_0/A:col10/1733740992261/Put/seqid=0 2024-12-09T10:43:12,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742343_1519 (size=30955) 2024-12-09T10:43:12,912 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:12,912 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:12,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60502 deadline: 1733741052910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:12,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60464 deadline: 1733741052909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:12,913 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:12,913 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:12,913 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:12,913 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:12,913 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:12,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60518 deadline: 1733741052910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:12,913 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:12,913 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:12,913 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:12,913 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:12,913 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:12,923 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:12,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60526 deadline: 1733741052920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:12,923 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:12,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60494 deadline: 1733741052920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:12,924 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:12,924 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:12,924 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:12,924 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:12,924 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:12,924 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:13,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-09T10:43:13,090 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T10:43:13,090 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-09T10:43:13,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-12-09T10:43:13,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-09T10:43:13,092 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T10:43:13,093 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T10:43:13,093 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T10:43:13,107 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=55, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/A/adc7d04859af409a8392ab390d0db49c 2024-12-09T10:43:13,114 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/B/dc3d6d4bf80e4ee49d99ed2eff703360 is 50, key is test_row_0/B:col10/1733740992261/Put/seqid=0 2024-12-09T10:43:13,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742344_1520 (size=12001) 2024-12-09T10:43:13,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-09T10:43:13,245 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-09T10:43:13,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:13,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. as already flushing 2024-12-09T10:43:13,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:13,245 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:43:13,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:43:13,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:43:13,398 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-09T10:43:13,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:13,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. as already flushing 2024-12-09T10:43:13,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:13,398 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:43:13,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:43:13,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:43:13,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-09T10:43:13,422 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:13,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60464 deadline: 1733741053419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:13,423 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:13,423 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:13,423 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:13,423 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:13,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60502 deadline: 1733741053419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:13,423 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:13,423 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:13,423 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:13,423 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:13,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60518 deadline: 1733741053419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:13,423 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:13,423 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:13,423 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:13,432 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:13,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60526 deadline: 1733741053430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:13,432 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:13,433 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:13,433 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:13,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60494 deadline: 1733741053430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:13,433 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:13,433 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:13,433 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:13,433 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:13,518 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/B/dc3d6d4bf80e4ee49d99ed2eff703360 2024-12-09T10:43:13,525 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/C/572b6eea657540d0ad6c8541f603d2de is 50, key is test_row_0/C:col10/1733740992261/Put/seqid=0 2024-12-09T10:43:13,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742345_1521 (size=12001) 2024-12-09T10:43:13,551 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-09T10:43:13,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:13,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. as already flushing 2024-12-09T10:43:13,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:13,551 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:43:13,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:43:13,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:43:13,704 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-09T10:43:13,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:13,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. as already flushing 2024-12-09T10:43:13,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:13,704 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:43:13,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:43:13,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:43:13,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-09T10:43:13,857 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-09T10:43:13,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:13,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. as already flushing 2024-12-09T10:43:13,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:13,857 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:43:13,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:43:13,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:43:13,929 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/C/572b6eea657540d0ad6c8541f603d2de 2024-12-09T10:43:13,933 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/A/adc7d04859af409a8392ab390d0db49c as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/adc7d04859af409a8392ab390d0db49c 2024-12-09T10:43:13,936 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/adc7d04859af409a8392ab390d0db49c, entries=150, sequenceid=55, filesize=30.2 K 2024-12-09T10:43:13,937 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/B/dc3d6d4bf80e4ee49d99ed2eff703360 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/dc3d6d4bf80e4ee49d99ed2eff703360 2024-12-09T10:43:13,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,940 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/dc3d6d4bf80e4ee49d99ed2eff703360, entries=150, sequenceid=55, filesize=11.7 K 2024-12-09T10:43:13,941 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/C/572b6eea657540d0ad6c8541f603d2de as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/572b6eea657540d0ad6c8541f603d2de 2024-12-09T10:43:13,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,944 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/572b6eea657540d0ad6c8541f603d2de, entries=150, sequenceid=55, filesize=11.7 K 2024-12-09T10:43:13,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,945 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 92c0b293dd887d05ae6f705c47b9c951 in 1682ms, sequenceid=55, compaction requested=true 2024-12-09T10:43:13,945 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 92c0b293dd887d05ae6f705c47b9c951: 2024-12-09T10:43:13,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 92c0b293dd887d05ae6f705c47b9c951:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T10:43:13,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:43:13,945 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T10:43:13,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 92c0b293dd887d05ae6f705c47b9c951:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T10:43:13,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:43:13,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 92c0b293dd887d05ae6f705c47b9c951:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T10:43:13,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T10:43:13,945 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T10:43:13,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,946 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T10:43:13,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,946 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1541): 92c0b293dd887d05ae6f705c47b9c951/A is initiating minor compaction (all files) 2024-12-09T10:43:13,946 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 92c0b293dd887d05ae6f705c47b9c951/A in TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:13,946 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/dc7da36494e34fb1921749692776ea6f, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/23b8ba829cf343628caa96e45fe9d3d2, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/adc7d04859af409a8392ab390d0db49c] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp, totalSize=90.7 K 2024-12-09T10:43:13,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,946 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:13,946 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. files: [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/dc7da36494e34fb1921749692776ea6f, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/23b8ba829cf343628caa96e45fe9d3d2, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/adc7d04859af409a8392ab390d0db49c] 2024-12-09T10:43:13,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,946 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T10:43:13,946 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1541): 92c0b293dd887d05ae6f705c47b9c951/B is initiating minor compaction (all files) 2024-12-09T10:43:13,946 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 92c0b293dd887d05ae6f705c47b9c951/B in TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:13,947 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/e252a9a06c3c47608ddaa35c36e798bc, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/ce55eca3e6e640cb9d810de214302669, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/dc3d6d4bf80e4ee49d99ed2eff703360] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp, totalSize=35.2 K 2024-12-09T10:43:13,947 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting dc7da36494e34fb1921749692776ea6f, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=22, earliestPutTs=1733740988931 2024-12-09T10:43:13,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,947 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting e252a9a06c3c47608ddaa35c36e798bc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=22, earliestPutTs=1733740988931 2024-12-09T10:43:13,947 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 23b8ba829cf343628caa96e45fe9d3d2, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1733740988944 2024-12-09T10:43:13,947 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting adc7d04859af409a8392ab390d0db49c, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1733740991120 2024-12-09T10:43:13,947 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting ce55eca3e6e640cb9d810de214302669, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1733740988944 2024-12-09T10:43:13,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,947 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting dc3d6d4bf80e4ee49d99ed2eff703360, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1733740991120 2024-12-09T10:43:13,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,952 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=92c0b293dd887d05ae6f705c47b9c951] 2024-12-09T10:43:13,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,954 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 92c0b293dd887d05ae6f705c47b9c951#B#compaction#444 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:43:13,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,955 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/B/0de7deafa4b047f5955b9d92716b7d8b is 50, key is test_row_0/B:col10/1733740992261/Put/seqid=0 2024-12-09T10:43:13,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,955 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241209fb46042ae93642d6b489cefa3edc553b_92c0b293dd887d05ae6f705c47b9c951 store=[table=TestAcidGuarantees family=A region=92c0b293dd887d05ae6f705c47b9c951] 2024-12-09T10:43:13,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,957 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241209fb46042ae93642d6b489cefa3edc553b_92c0b293dd887d05ae6f705c47b9c951, store=[table=TestAcidGuarantees family=A region=92c0b293dd887d05ae6f705c47b9c951] 2024-12-09T10:43:13,957 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209fb46042ae93642d6b489cefa3edc553b_92c0b293dd887d05ae6f705c47b9c951 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=92c0b293dd887d05ae6f705c47b9c951] 2024-12-09T10:43:13,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742347_1523 (size=4469) 2024-12-09T10:43:13,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742346_1522 (size=12104) 2024-12-09T10:43:13,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:13,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,010 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-09T10:43:14,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:14,010 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing 92c0b293dd887d05ae6f705c47b9c951 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-09T10:43:14,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 92c0b293dd887d05ae6f705c47b9c951, store=A 2024-12-09T10:43:14,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:43:14,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 92c0b293dd887d05ae6f705c47b9c951, store=B 2024-12-09T10:43:14,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:43:14,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 92c0b293dd887d05ae6f705c47b9c951, store=C 2024-12-09T10:43:14,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:43:14,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209715d91b173ef41c8a5b7e7fca008514b_92c0b293dd887d05ae6f705c47b9c951 is 50, key is test_row_0/A:col10/1733740992282/Put/seqid=0 2024-12-09T10:43:14,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742348_1524 (size=9714) 2024-12-09T10:43:14,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-09T10:43:14,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,362 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 92c0b293dd887d05ae6f705c47b9c951#A#compaction#443 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:43:14,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,363 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/A/d60dd66537d740f69f107cffc071d6aa is 175, key is test_row_0/A:col10/1733740992261/Put/seqid=0 2024-12-09T10:43:14,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742349_1525 (size=31058) 2024-12-09T10:43:14,369 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/B/0de7deafa4b047f5955b9d92716b7d8b as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/0de7deafa4b047f5955b9d92716b7d8b 2024-12-09T10:43:14,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,374 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 92c0b293dd887d05ae6f705c47b9c951/B of 92c0b293dd887d05ae6f705c47b9c951 into 0de7deafa4b047f5955b9d92716b7d8b(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:43:14,374 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 92c0b293dd887d05ae6f705c47b9c951: 2024-12-09T10:43:14,374 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., storeName=92c0b293dd887d05ae6f705c47b9c951/B, priority=13, startTime=1733740993945; duration=0sec 2024-12-09T10:43:14,374 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T10:43:14,374 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 92c0b293dd887d05ae6f705c47b9c951:B 2024-12-09T10:43:14,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,374 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T10:43:14,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,375 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T10:43:14,375 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1541): 92c0b293dd887d05ae6f705c47b9c951/C is initiating minor compaction (all files) 2024-12-09T10:43:14,375 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 92c0b293dd887d05ae6f705c47b9c951/C in TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:14,375 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/3a9d00eb495849c59b9168e4d88183d2, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/b5d7a7ca62f34f16908ba630a7886775, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/572b6eea657540d0ad6c8541f603d2de] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp, totalSize=35.2 K 2024-12-09T10:43:14,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,375 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a9d00eb495849c59b9168e4d88183d2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=22, earliestPutTs=1733740988931 2024-12-09T10:43:14,376 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting b5d7a7ca62f34f16908ba630a7886775, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1733740988944 2024-12-09T10:43:14,376 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 572b6eea657540d0ad6c8541f603d2de, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1733740991120 2024-12-09T10:43:14,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,387 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 92c0b293dd887d05ae6f705c47b9c951#C#compaction#446 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:43:14,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,388 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/C/8862f157d1e349e09e21b281245e6783 is 50, key is test_row_0/C:col10/1733740992261/Put/seqid=0 2024-12-09T10:43:14,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742350_1526 (size=12104) 2024-12-09T10:43:14,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,426 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209715d91b173ef41c8a5b7e7fca008514b_92c0b293dd887d05ae6f705c47b9c951 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209715d91b173ef41c8a5b7e7fca008514b_92c0b293dd887d05ae6f705c47b9c951 2024-12-09T10:43:14,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/A/5f82811dc8b04a4e83fd1f45bf809ed1, store: [table=TestAcidGuarantees family=A region=92c0b293dd887d05ae6f705c47b9c951] 2024-12-09T10:43:14,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/A/5f82811dc8b04a4e83fd1f45bf809ed1 is 175, key is test_row_0/A:col10/1733740992282/Put/seqid=0 2024-12-09T10:43:14,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742351_1527 (size=22361) 2024-12-09T10:43:14,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,432 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=68, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/A/5f82811dc8b04a4e83fd1f45bf809ed1 2024-12-09T10:43:14,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(8855): Flush requested on 92c0b293dd887d05ae6f705c47b9c951 2024-12-09T10:43:14,436 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. as already flushing 2024-12-09T10:43:14,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:14,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/B/3597e8a7c1774b9a8da80008b00ea2d2 is 50, key is test_row_0/B:col10/1733740992282/Put/seqid=0 2024-12-09T10:43:14,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742352_1528 (size=9657) 2024-12-09T10:43:14,446 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=68 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/B/3597e8a7c1774b9a8da80008b00ea2d2 2024-12-09T10:43:14,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/C/066f76192f194b41a89f3c2381b8c17d is 50, key is test_row_0/C:col10/1733740992282/Put/seqid=0 2024-12-09T10:43:14,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742353_1529 (size=9657) 2024-12-09T10:43:14,460 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:14,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60518 deadline: 1733741054455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:14,461 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:14,461 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:14,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60464 deadline: 1733741054457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:14,461 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:14,461 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:14,461 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:14,461 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:14,461 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:14,464 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:14,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60502 deadline: 1733741054459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:14,464 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:14,464 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:14,464 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:14,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60526 deadline: 1733741054459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:14,464 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:14,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60494 deadline: 1733741054460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:14,464 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:14,465 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:14,465 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:14,465 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:14,465 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:14,465 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:14,465 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:14,571 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:14,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60518 deadline: 1733741054569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:14,572 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:14,572 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:14,572 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:14,572 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:14,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60464 deadline: 1733741054570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:14,572 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:14,572 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:14,572 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:14,572 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:14,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60502 deadline: 1733741054570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:14,572 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:14,572 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:14,572 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:14,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60494 deadline: 1733741054570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:14,572 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:14,573 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:14,573 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:14,573 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:14,573 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:14,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60526 deadline: 1733741054570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:14,573 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:14,573 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:14,573 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:14,773 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/A/d60dd66537d740f69f107cffc071d6aa as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/d60dd66537d740f69f107cffc071d6aa 2024-12-09T10:43:14,777 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 92c0b293dd887d05ae6f705c47b9c951/A of 92c0b293dd887d05ae6f705c47b9c951 into d60dd66537d740f69f107cffc071d6aa(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:43:14,777 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 92c0b293dd887d05ae6f705c47b9c951: 2024-12-09T10:43:14,777 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., storeName=92c0b293dd887d05ae6f705c47b9c951/A, priority=13, startTime=1733740993945; duration=0sec 2024-12-09T10:43:14,777 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:43:14,777 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 92c0b293dd887d05ae6f705c47b9c951:A 2024-12-09T10:43:14,780 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:14,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60518 deadline: 1733741054780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:14,781 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:14,781 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:14,781 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:14,781 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:14,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60464 deadline: 1733741054780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:14,781 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:14,781 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:14,781 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:14,781 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:14,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60494 deadline: 1733741054780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:14,782 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:14,782 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:14,782 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:14,782 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:14,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60502 deadline: 1733741054780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:14,782 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:14,782 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:14,782 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:14,782 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:14,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60526 deadline: 1733741054780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:14,782 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:14,782 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:14,782 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:14,799 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/C/8862f157d1e349e09e21b281245e6783 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/8862f157d1e349e09e21b281245e6783 2024-12-09T10:43:14,803 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 92c0b293dd887d05ae6f705c47b9c951/C of 92c0b293dd887d05ae6f705c47b9c951 into 8862f157d1e349e09e21b281245e6783(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:43:14,803 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 92c0b293dd887d05ae6f705c47b9c951: 2024-12-09T10:43:14,803 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., storeName=92c0b293dd887d05ae6f705c47b9c951/C, priority=13, startTime=1733740993945; duration=0sec 2024-12-09T10:43:14,803 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:43:14,803 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 92c0b293dd887d05ae6f705c47b9c951:C 2024-12-09T10:43:14,857 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=68 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/C/066f76192f194b41a89f3c2381b8c17d 2024-12-09T10:43:14,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/A/5f82811dc8b04a4e83fd1f45bf809ed1 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/5f82811dc8b04a4e83fd1f45bf809ed1 2024-12-09T10:43:14,868 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/5f82811dc8b04a4e83fd1f45bf809ed1, entries=100, sequenceid=68, filesize=21.8 K 2024-12-09T10:43:14,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/B/3597e8a7c1774b9a8da80008b00ea2d2 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/3597e8a7c1774b9a8da80008b00ea2d2 2024-12-09T10:43:14,871 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/3597e8a7c1774b9a8da80008b00ea2d2, entries=100, sequenceid=68, filesize=9.4 K 2024-12-09T10:43:14,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/C/066f76192f194b41a89f3c2381b8c17d as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/066f76192f194b41a89f3c2381b8c17d 2024-12-09T10:43:14,877 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/066f76192f194b41a89f3c2381b8c17d, entries=100, sequenceid=68, filesize=9.4 K 2024-12-09T10:43:14,886 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 92c0b293dd887d05ae6f705c47b9c951 in 875ms, sequenceid=68, compaction requested=false 2024-12-09T10:43:14,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for 92c0b293dd887d05ae6f705c47b9c951: 2024-12-09T10:43:14,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:14,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-12-09T10:43:14,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-12-09T10:43:14,889 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-12-09T10:43:14,889 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7940 sec 2024-12-09T10:43:14,891 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 1.8000 sec 2024-12-09T10:43:15,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(8855): Flush requested on 92c0b293dd887d05ae6f705c47b9c951 2024-12-09T10:43:15,093 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 92c0b293dd887d05ae6f705c47b9c951 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-09T10:43:15,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 92c0b293dd887d05ae6f705c47b9c951, store=A 2024-12-09T10:43:15,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T10:43:15,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 92c0b293dd887d05ae6f705c47b9c951, store=B 2024-12-09T10:43:15,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T10:43:15,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 92c0b293dd887d05ae6f705c47b9c951, store=C 2024-12-09T10:43:15,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T10:43:15,102 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209862d65847d0f414ba3ceec3729fbe1a9_92c0b293dd887d05ae6f705c47b9c951 is 50, key is test_row_0/A:col10/1733740995092/Put/seqid=0 2024-12-09T10:43:15,103 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:15,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60518 deadline: 1733741055099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:15,104 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:15,104 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:15,104 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:15,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742354_1530 (size=12154) 2024-12-09T10:43:15,106 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:15,110 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:15,110 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:15,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60502 deadline: 1733741055102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:15,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60464 deadline: 1733741055103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:15,111 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:15,111 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:15,111 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:15,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60526 deadline: 1733741055104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:15,111 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:15,111 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:15,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60494 deadline: 1733741055104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:15,112 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:15,112 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:15,112 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:15,112 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209862d65847d0f414ba3ceec3729fbe1a9_92c0b293dd887d05ae6f705c47b9c951 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209862d65847d0f414ba3ceec3729fbe1a9_92c0b293dd887d05ae6f705c47b9c951 2024-12-09T10:43:15,113 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:15,113 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:15,113 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:15,113 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/A/1e1c6f08f05b4a87ba4205353ba6a9c8, store: [table=TestAcidGuarantees family=A region=92c0b293dd887d05ae6f705c47b9c951] 2024-12-09T10:43:15,114 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/A/1e1c6f08f05b4a87ba4205353ba6a9c8 is 175, key is test_row_0/A:col10/1733740995092/Put/seqid=0 2024-12-09T10:43:15,118 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:15,118 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:15,118 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:15,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742355_1531 (size=30955) 2024-12-09T10:43:15,132 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=94, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/A/1e1c6f08f05b4a87ba4205353ba6a9c8 2024-12-09T10:43:15,150 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/B/36d31d8e091f4f8b8f0c295903d9e83b is 50, key is test_row_0/B:col10/1733740995092/Put/seqid=0 2024-12-09T10:43:15,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742356_1532 (size=12001) 2024-12-09T10:43:15,168 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/B/36d31d8e091f4f8b8f0c295903d9e83b 2024-12-09T10:43:15,180 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/C/fc1dfdec982841ffb0f9d4b70e4fe590 is 50, key is test_row_0/C:col10/1733740995092/Put/seqid=0 2024-12-09T10:43:15,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742357_1533 (size=12001) 2024-12-09T10:43:15,212 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:15,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60518 deadline: 1733741055210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:15,212 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:15,212 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:15,212 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:15,221 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:15,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60464 deadline: 1733741055219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:15,222 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:15,222 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:15,222 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:15,222 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:15,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60494 deadline: 1733741055219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:15,222 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:15,222 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:15,222 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:15,222 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:15,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60526 deadline: 1733741055219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:15,222 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:15,222 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:15,222 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:15,222 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:15,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60502 deadline: 1733741055219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:15,223 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:15,223 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:15,223 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:15,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-09T10:43:15,250 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T10:43:15,254 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-09T10:43:15,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-12-09T10:43:15,256 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T10:43:15,257 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T10:43:15,257 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T10:43:15,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-09T10:43:15,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-09T10:43:15,410 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-09T10:43:15,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:15,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. as already flushing 2024-12-09T10:43:15,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:15,411 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:43:15,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:43:15,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:43:15,424 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:15,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60518 deadline: 1733741055420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:15,425 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:15,425 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:15,425 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:15,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:15,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60464 deadline: 1733741055430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:15,434 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:15,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60494 deadline: 1733741055430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:15,434 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:15,434 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:15,434 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:15,434 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:15,435 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:15,435 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:15,435 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:15,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60526 deadline: 1733741055431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:15,435 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:15,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60502 deadline: 1733741055431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:15,435 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:15,435 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:15,435 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:15,436 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:15,436 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:15,436 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:15,565 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-09T10:43:15,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:15,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. as already flushing 2024-12-09T10:43:15,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:15,566 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:43:15,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:43:15,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:43:15,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-09T10:43:15,596 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/C/fc1dfdec982841ffb0f9d4b70e4fe590 2024-12-09T10:43:15,603 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/A/1e1c6f08f05b4a87ba4205353ba6a9c8 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/1e1c6f08f05b4a87ba4205353ba6a9c8 2024-12-09T10:43:15,613 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/1e1c6f08f05b4a87ba4205353ba6a9c8, entries=150, sequenceid=94, filesize=30.2 K 2024-12-09T10:43:15,616 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/B/36d31d8e091f4f8b8f0c295903d9e83b as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/36d31d8e091f4f8b8f0c295903d9e83b 2024-12-09T10:43:15,623 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/36d31d8e091f4f8b8f0c295903d9e83b, entries=150, sequenceid=94, filesize=11.7 K 2024-12-09T10:43:15,625 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/C/fc1dfdec982841ffb0f9d4b70e4fe590 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/fc1dfdec982841ffb0f9d4b70e4fe590 2024-12-09T10:43:15,635 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/fc1dfdec982841ffb0f9d4b70e4fe590, entries=150, sequenceid=94, filesize=11.7 K 2024-12-09T10:43:15,636 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~399.84 KB/409440, currentSize=40.25 KB/41220 for 92c0b293dd887d05ae6f705c47b9c951 in 543ms, sequenceid=94, compaction requested=true 2024-12-09T10:43:15,636 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 92c0b293dd887d05ae6f705c47b9c951: 2024-12-09T10:43:15,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 92c0b293dd887d05ae6f705c47b9c951:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T10:43:15,636 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T10:43:15,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:43:15,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 92c0b293dd887d05ae6f705c47b9c951:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T10:43:15,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:43:15,636 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T10:43:15,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 92c0b293dd887d05ae6f705c47b9c951:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T10:43:15,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T10:43:15,637 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 84374 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T10:43:15,637 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1541): 92c0b293dd887d05ae6f705c47b9c951/A is initiating minor compaction (all files) 2024-12-09T10:43:15,637 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 92c0b293dd887d05ae6f705c47b9c951/A in TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:15,637 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/d60dd66537d740f69f107cffc071d6aa, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/5f82811dc8b04a4e83fd1f45bf809ed1, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/1e1c6f08f05b4a87ba4205353ba6a9c8] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp, totalSize=82.4 K 2024-12-09T10:43:15,637 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:15,637 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. files: [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/d60dd66537d740f69f107cffc071d6aa, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/5f82811dc8b04a4e83fd1f45bf809ed1, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/1e1c6f08f05b4a87ba4205353ba6a9c8] 2024-12-09T10:43:15,639 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33762 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T10:43:15,639 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1541): 92c0b293dd887d05ae6f705c47b9c951/B is initiating minor compaction (all files) 2024-12-09T10:43:15,639 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 92c0b293dd887d05ae6f705c47b9c951/B in TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:15,639 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/0de7deafa4b047f5955b9d92716b7d8b, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/3597e8a7c1774b9a8da80008b00ea2d2, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/36d31d8e091f4f8b8f0c295903d9e83b] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp, totalSize=33.0 K 2024-12-09T10:43:15,639 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting d60dd66537d740f69f107cffc071d6aa, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1733740991120 2024-12-09T10:43:15,640 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 0de7deafa4b047f5955b9d92716b7d8b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1733740991120 2024-12-09T10:43:15,640 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 3597e8a7c1774b9a8da80008b00ea2d2, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1733740992282 2024-12-09T10:43:15,640 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5f82811dc8b04a4e83fd1f45bf809ed1, keycount=100, bloomtype=ROW, size=21.8 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1733740992282 2024-12-09T10:43:15,640 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e1c6f08f05b4a87ba4205353ba6a9c8, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1733740995092 2024-12-09T10:43:15,640 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 36d31d8e091f4f8b8f0c295903d9e83b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1733740995092 2024-12-09T10:43:15,648 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 92c0b293dd887d05ae6f705c47b9c951#B#compaction#452 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:43:15,648 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/B/3c04b67b06674529bc752e4cb546cd38 is 50, key is test_row_0/B:col10/1733740995092/Put/seqid=0 2024-12-09T10:43:15,666 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=92c0b293dd887d05ae6f705c47b9c951] 2024-12-09T10:43:15,699 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120940fad975dd164cd0b1374480f98c3a74_92c0b293dd887d05ae6f705c47b9c951 store=[table=TestAcidGuarantees family=A region=92c0b293dd887d05ae6f705c47b9c951] 2024-12-09T10:43:15,701 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120940fad975dd164cd0b1374480f98c3a74_92c0b293dd887d05ae6f705c47b9c951, store=[table=TestAcidGuarantees family=A region=92c0b293dd887d05ae6f705c47b9c951] 2024-12-09T10:43:15,701 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120940fad975dd164cd0b1374480f98c3a74_92c0b293dd887d05ae6f705c47b9c951 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=92c0b293dd887d05ae6f705c47b9c951] 2024-12-09T10:43:15,719 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-09T10:43:15,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:15,720 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing 92c0b293dd887d05ae6f705c47b9c951 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-12-09T10:43:15,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 92c0b293dd887d05ae6f705c47b9c951, store=A 2024-12-09T10:43:15,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:43:15,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 92c0b293dd887d05ae6f705c47b9c951, store=B 2024-12-09T10:43:15,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:43:15,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 92c0b293dd887d05ae6f705c47b9c951, store=C 2024-12-09T10:43:15,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:43:15,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742358_1534 (size=12207) 2024-12-09T10:43:15,731 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/B/3c04b67b06674529bc752e4cb546cd38 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/3c04b67b06674529bc752e4cb546cd38 2024-12-09T10:43:15,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(8855): Flush requested on 92c0b293dd887d05ae6f705c47b9c951 2024-12-09T10:43:15,734 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. as already flushing 2024-12-09T10:43:15,739 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 92c0b293dd887d05ae6f705c47b9c951/B of 92c0b293dd887d05ae6f705c47b9c951 into 3c04b67b06674529bc752e4cb546cd38(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:43:15,739 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 92c0b293dd887d05ae6f705c47b9c951: 2024-12-09T10:43:15,739 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., storeName=92c0b293dd887d05ae6f705c47b9c951/B, priority=13, startTime=1733740995636; duration=0sec 2024-12-09T10:43:15,739 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T10:43:15,739 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 92c0b293dd887d05ae6f705c47b9c951:B 2024-12-09T10:43:15,739 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T10:43:15,743 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33762 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T10:43:15,743 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1541): 92c0b293dd887d05ae6f705c47b9c951/C is initiating minor compaction (all files) 2024-12-09T10:43:15,743 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 92c0b293dd887d05ae6f705c47b9c951/C in TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:15,744 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/8862f157d1e349e09e21b281245e6783, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/066f76192f194b41a89f3c2381b8c17d, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/fc1dfdec982841ffb0f9d4b70e4fe590] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp, totalSize=33.0 K 2024-12-09T10:43:15,748 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 8862f157d1e349e09e21b281245e6783, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1733740991120 2024-12-09T10:43:15,749 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 066f76192f194b41a89f3c2381b8c17d, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1733740992282 2024-12-09T10:43:15,749 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting fc1dfdec982841ffb0f9d4b70e4fe590, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1733740995092 2024-12-09T10:43:15,780 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733740995763, max timestamp=1733740995776Num uniques 859; 2024-12-09T10:43:15,780 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T10:43:15,780 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733740995763, max timestamp=1733740995776Num uniques 859; 2024-12-09T10:43:15,780 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T10:43:15,784 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733740995763, max timestamp=1733740995776Num uniques 859; 2024-12-09T10:43:15,784 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T10:43:15,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742359_1535 (size=4469) 2024-12-09T10:43:15,787 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 92c0b293dd887d05ae6f705c47b9c951#A#compaction#453 average throughput is 0.20 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:43:15,788 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/A/7f5217c3ade4443d9117f24a7ec728be is 175, key is test_row_0/A:col10/1733740995092/Put/seqid=0 2024-12-09T10:43:15,791 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 92c0b293dd887d05ae6f705c47b9c951#C#compaction#455 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:43:15,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209f899c477f016441d963248af158a6dcc_92c0b293dd887d05ae6f705c47b9c951 is 50, key is test_row_0/A:col10/1733740995102/Put/seqid=0 2024-12-09T10:43:15,792 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/C/59f7b5186ffe4c2dad6b664d6e1dc9b8 is 50, key is test_row_0/C:col10/1733740995092/Put/seqid=0 2024-12-09T10:43:15,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:15,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60494 deadline: 1733741055818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:15,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742360_1536 (size=31161) 2024-12-09T10:43:15,831 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:15,831 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:15,831 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:15,838 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1733740995810, max timestamp=1733740995830Num uniques 1068; 2024-12-09T10:43:15,838 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1733740995810, max timestamp=1733740995830Num uniques 1068; 2024-12-09T10:43:15,838 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=14558, suffix heap size=139040,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=318 2024-12-09T10:43:15,838 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1733740995810, max timestamp=1733740995830Num uniques 1018; 2024-12-09T10:43:15,838 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=14558, suffix heap size=139040,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=318 2024-12-09T10:43:15,838 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=14558, suffix heap size=133840,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=318 2024-12-09T10:43:15,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742362_1538 (size=12207) 2024-12-09T10:43:15,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742361_1537 (size=12154) 2024-12-09T10:43:15,867 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/C/59f7b5186ffe4c2dad6b664d6e1dc9b8 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/59f7b5186ffe4c2dad6b664d6e1dc9b8 2024-12-09T10:43:15,872 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:15,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60464 deadline: 1733741055864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:15,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:15,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:15,873 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:15,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:15,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60526 deadline: 1733741055865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:15,874 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:15,874 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:15,874 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:15,875 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 92c0b293dd887d05ae6f705c47b9c951/C of 92c0b293dd887d05ae6f705c47b9c951 into 59f7b5186ffe4c2dad6b664d6e1dc9b8(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:43:15,875 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 92c0b293dd887d05ae6f705c47b9c951: 2024-12-09T10:43:15,875 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., storeName=92c0b293dd887d05ae6f705c47b9c951/C, priority=13, startTime=1733740995636; duration=0sec 2024-12-09T10:43:15,875 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:43:15,875 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 92c0b293dd887d05ae6f705c47b9c951:C 2024-12-09T10:43:15,877 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:15,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60518 deadline: 1733741055871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:15,877 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:15,877 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:15,877 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:15,881 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:15,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60502 deadline: 1733741055873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:15,882 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:15,882 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:15,882 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:15,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-09T10:43:15,942 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:15,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60494 deadline: 1733741055939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:15,943 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:15,943 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:15,943 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:15,983 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:15,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60464 deadline: 1733741055980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:15,984 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:15,984 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:15,984 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:15,985 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:15,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60518 deadline: 1733741055980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:15,986 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:15,986 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:15,986 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:15,988 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:15,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60526 deadline: 1733741055986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:15,989 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:15,989 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:15,989 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:15,995 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:15,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60502 deadline: 1733741055989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:15,996 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:15,996 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:15,996 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:16,150 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:16,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60494 deadline: 1733741056149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:16,151 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:16,151 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:16,151 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:16,191 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:16,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60464 deadline: 1733741056190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:16,192 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:16,192 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:16,192 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:16,192 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:16,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60518 deadline: 1733741056190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:16,192 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:16,192 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:16,192 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:16,204 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:16,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60526 deadline: 1733741056200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:16,205 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:16,205 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:16,205 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:16,205 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:16,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60502 deadline: 1733741056200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:16,206 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:16,206 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:16,206 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:16,244 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/A/7f5217c3ade4443d9117f24a7ec728be as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/7f5217c3ade4443d9117f24a7ec728be 2024-12-09T10:43:16,249 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 92c0b293dd887d05ae6f705c47b9c951/A of 92c0b293dd887d05ae6f705c47b9c951 into 7f5217c3ade4443d9117f24a7ec728be(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:43:16,249 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 92c0b293dd887d05ae6f705c47b9c951: 2024-12-09T10:43:16,249 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., storeName=92c0b293dd887d05ae6f705c47b9c951/A, priority=13, startTime=1733740995636; duration=0sec 2024-12-09T10:43:16,249 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:43:16,249 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 92c0b293dd887d05ae6f705c47b9c951:A 2024-12-09T10:43:16,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:16,268 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209f899c477f016441d963248af158a6dcc_92c0b293dd887d05ae6f705c47b9c951 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209f899c477f016441d963248af158a6dcc_92c0b293dd887d05ae6f705c47b9c951 2024-12-09T10:43:16,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/A/7d4be0511f014f8e95f988850c2d3c15, store: [table=TestAcidGuarantees family=A region=92c0b293dd887d05ae6f705c47b9c951] 2024-12-09T10:43:16,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/A/7d4be0511f014f8e95f988850c2d3c15 is 175, key is test_row_0/A:col10/1733740995102/Put/seqid=0 2024-12-09T10:43:16,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742363_1539 (size=30955) 2024-12-09T10:43:16,300 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=103, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/A/7d4be0511f014f8e95f988850c2d3c15 2024-12-09T10:43:16,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/B/6192782f6c42497094580729231f4a4d is 50, key is test_row_0/B:col10/1733740995102/Put/seqid=0 2024-12-09T10:43:16,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742364_1540 (size=12001) 2024-12-09T10:43:16,329 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/B/6192782f6c42497094580729231f4a4d 2024-12-09T10:43:16,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/C/faff8273cb8048ddbae1d87d05b35d96 is 50, key is test_row_0/C:col10/1733740995102/Put/seqid=0 2024-12-09T10:43:16,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742365_1541 (size=12001) 2024-12-09T10:43:16,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-09T10:43:16,464 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:16,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60494 deadline: 1733741056460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:16,464 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:16,464 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:16,464 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:16,503 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:16,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60464 deadline: 1733741056499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:16,504 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:16,504 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:16,504 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:16,504 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:16,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60518 deadline: 1733741056499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:16,504 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:16,504 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:16,504 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:16,514 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:16,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60526 deadline: 1733741056510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:16,514 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:16,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60502 deadline: 1733741056510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:16,515 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:16,515 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:16,515 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:16,515 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:16,515 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:16,515 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:16,740 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/C/faff8273cb8048ddbae1d87d05b35d96 2024-12-09T10:43:16,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/A/7d4be0511f014f8e95f988850c2d3c15 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/7d4be0511f014f8e95f988850c2d3c15 2024-12-09T10:43:16,748 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/7d4be0511f014f8e95f988850c2d3c15, entries=150, sequenceid=103, filesize=30.2 K 2024-12-09T10:43:16,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/B/6192782f6c42497094580729231f4a4d as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/6192782f6c42497094580729231f4a4d 2024-12-09T10:43:16,751 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/6192782f6c42497094580729231f4a4d, entries=150, sequenceid=103, filesize=11.7 K 2024-12-09T10:43:16,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/C/faff8273cb8048ddbae1d87d05b35d96 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/faff8273cb8048ddbae1d87d05b35d96 2024-12-09T10:43:16,755 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/faff8273cb8048ddbae1d87d05b35d96, entries=150, sequenceid=103, filesize=11.7 K 2024-12-09T10:43:16,756 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=147.60 KB/151140 for 92c0b293dd887d05ae6f705c47b9c951 in 1037ms, sequenceid=103, compaction requested=false 2024-12-09T10:43:16,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for 92c0b293dd887d05ae6f705c47b9c951: 2024-12-09T10:43:16,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:16,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-12-09T10:43:16,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-12-09T10:43:16,759 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-12-09T10:43:16,759 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5000 sec 2024-12-09T10:43:16,761 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 1.5060 sec 2024-12-09T10:43:16,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(8855): Flush requested on 92c0b293dd887d05ae6f705c47b9c951 2024-12-09T10:43:16,973 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 92c0b293dd887d05ae6f705c47b9c951 3/3 column families, dataSize=154.31 KB heapSize=429.84 KB 2024-12-09T10:43:16,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 92c0b293dd887d05ae6f705c47b9c951, store=A 2024-12-09T10:43:16,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T10:43:16,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 92c0b293dd887d05ae6f705c47b9c951, store=B 2024-12-09T10:43:16,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T10:43:16,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 92c0b293dd887d05ae6f705c47b9c951, store=C 2024-12-09T10:43:16,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T10:43:16,981 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209d3fe4b6375d647c79741cafd22e2ed9b_92c0b293dd887d05ae6f705c47b9c951 is 50, key is test_row_0/A:col10/1733740996972/Put/seqid=0 2024-12-09T10:43:16,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742366_1542 (size=14794) 2024-12-09T10:43:16,986 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:16,991 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209d3fe4b6375d647c79741cafd22e2ed9b_92c0b293dd887d05ae6f705c47b9c951 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209d3fe4b6375d647c79741cafd22e2ed9b_92c0b293dd887d05ae6f705c47b9c951 2024-12-09T10:43:16,994 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/A/ebc6b4168c9e416d85b123e7e41d1021, store: [table=TestAcidGuarantees family=A region=92c0b293dd887d05ae6f705c47b9c951] 2024-12-09T10:43:16,994 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/A/ebc6b4168c9e416d85b123e7e41d1021 is 175, key is test_row_0/A:col10/1733740996972/Put/seqid=0 2024-12-09T10:43:17,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742367_1543 (size=39749) 2024-12-09T10:43:17,004 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=160, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/A/ebc6b4168c9e416d85b123e7e41d1021 2024-12-09T10:43:17,010 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:17,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60494 deadline: 1733741057004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:17,010 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:17,010 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:17,010 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:17,010 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:17,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60464 deadline: 1733741057010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:17,011 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:17,011 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:17,011 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:17,011 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:17,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60518 deadline: 1733741057010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:17,011 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:17,011 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:17,011 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:17,016 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/B/97951b05faf74c9fbfbc59569daacaac is 50, key is test_row_0/B:col10/1733740996972/Put/seqid=0 2024-12-09T10:43:17,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742368_1544 (size=12151) 2024-12-09T10:43:17,023 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:17,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60526 deadline: 1733741057020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:17,024 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:17,024 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:17,024 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:17,030 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:17,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60502 deadline: 1733741057029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:17,030 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:17,030 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:17,030 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:17,121 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:17,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60494 deadline: 1733741057119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:17,121 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:17,122 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:17,122 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:17,336 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:17,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60494 deadline: 1733741057330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:17,336 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:17,336 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:17,336 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:17,343 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-09T10:43:17,343 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-12-09T10:43:17,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-09T10:43:17,420 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T10:43:17,421 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-09T10:43:17,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-12-09T10:43:17,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-09T10:43:17,424 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=160 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/B/97951b05faf74c9fbfbc59569daacaac 2024-12-09T10:43:17,424 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T10:43:17,425 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T10:43:17,425 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T10:43:17,444 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/C/03c22604e71f453fa94b830a8878ac51 is 50, key is test_row_0/C:col10/1733740996972/Put/seqid=0 2024-12-09T10:43:17,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742369_1545 (size=12151) 2024-12-09T10:43:17,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-09T10:43:17,578 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-09T10:43:17,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:17,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. as already flushing 2024-12-09T10:43:17,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:17,578 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:43:17,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:43:17,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:43:17,652 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:17,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60494 deadline: 1733741057649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:17,653 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:17,653 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:17,653 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:17,730 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-09T10:43:17,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:17,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. as already flushing 2024-12-09T10:43:17,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:17,731 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:43:17,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:43:17,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:43:17,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-09T10:43:17,862 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=160 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/C/03c22604e71f453fa94b830a8878ac51 2024-12-09T10:43:17,867 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/A/ebc6b4168c9e416d85b123e7e41d1021 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/ebc6b4168c9e416d85b123e7e41d1021 2024-12-09T10:43:17,870 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/ebc6b4168c9e416d85b123e7e41d1021, entries=200, sequenceid=160, filesize=38.8 K 2024-12-09T10:43:17,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,871 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/B/97951b05faf74c9fbfbc59569daacaac as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/97951b05faf74c9fbfbc59569daacaac 2024-12-09T10:43:17,875 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/97951b05faf74c9fbfbc59569daacaac, entries=150, sequenceid=160, filesize=11.9 K 2024-12-09T10:43:17,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,877 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/C/03c22604e71f453fa94b830a8878ac51 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/03c22604e71f453fa94b830a8878ac51 2024-12-09T10:43:17,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,881 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/03c22604e71f453fa94b830a8878ac51, entries=150, sequenceid=160, filesize=11.9 K 2024-12-09T10:43:17,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,883 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-09T10:43:17,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:17,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. as already flushing 2024-12-09T10:43:17,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:17,884 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:43:17,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:43:17,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:43:17,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,886 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~429.80 KB/440112, currentSize=33.54 KB/34350 for 92c0b293dd887d05ae6f705c47b9c951 in 913ms, sequenceid=160, compaction requested=true 2024-12-09T10:43:17,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,887 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 92c0b293dd887d05ae6f705c47b9c951: 2024-12-09T10:43:17,887 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T10:43:17,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,888 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T10:43:17,888 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1541): 92c0b293dd887d05ae6f705c47b9c951/A is initiating minor compaction (all files) 2024-12-09T10:43:17,888 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 92c0b293dd887d05ae6f705c47b9c951/A in TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:17,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,888 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/7f5217c3ade4443d9117f24a7ec728be, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/7d4be0511f014f8e95f988850c2d3c15, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/ebc6b4168c9e416d85b123e7e41d1021] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp, totalSize=99.5 K 2024-12-09T10:43:17,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,888 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:17,889 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. files: [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/7f5217c3ade4443d9117f24a7ec728be, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/7d4be0511f014f8e95f988850c2d3c15, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/ebc6b4168c9e416d85b123e7e41d1021] 2024-12-09T10:43:17,889 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f5217c3ade4443d9117f24a7ec728be, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1733740995092 2024-12-09T10:43:17,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 92c0b293dd887d05ae6f705c47b9c951:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T10:43:17,890 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7d4be0511f014f8e95f988850c2d3c15, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1733740995101 2024-12-09T10:43:17,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:43:17,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,890 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T10:43:17,890 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting ebc6b4168c9e416d85b123e7e41d1021, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1733740995857 2024-12-09T10:43:17,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 92c0b293dd887d05ae6f705c47b9c951:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T10:43:17,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:43:17,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 92c0b293dd887d05ae6f705c47b9c951:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T10:43:17,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T10:43:17,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,891 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T10:43:17,891 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1541): 92c0b293dd887d05ae6f705c47b9c951/B is initiating minor compaction (all files) 2024-12-09T10:43:17,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,891 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 92c0b293dd887d05ae6f705c47b9c951/B in TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:17,891 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/3c04b67b06674529bc752e4cb546cd38, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/6192782f6c42497094580729231f4a4d, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/97951b05faf74c9fbfbc59569daacaac] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp, totalSize=35.5 K 2024-12-09T10:43:17,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,892 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c04b67b06674529bc752e4cb546cd38, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1733740995092 2024-12-09T10:43:17,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,892 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 6192782f6c42497094580729231f4a4d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1733740995101 2024-12-09T10:43:17,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,892 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 97951b05faf74c9fbfbc59569daacaac, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1733740995857 2024-12-09T10:43:17,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,897 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=92c0b293dd887d05ae6f705c47b9c951] 2024-12-09T10:43:17,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,900 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 92c0b293dd887d05ae6f705c47b9c951#B#compaction#462 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:43:17,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,900 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/B/07072ac26e9b4995aba7c749e9c85570 is 50, key is test_row_0/B:col10/1733740996972/Put/seqid=0 2024-12-09T10:43:17,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,907 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412091fdfe00bd2c7424fadb91a0a11fcee79_92c0b293dd887d05ae6f705c47b9c951 store=[table=TestAcidGuarantees family=A region=92c0b293dd887d05ae6f705c47b9c951] 2024-12-09T10:43:17,909 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412091fdfe00bd2c7424fadb91a0a11fcee79_92c0b293dd887d05ae6f705c47b9c951, store=[table=TestAcidGuarantees family=A region=92c0b293dd887d05ae6f705c47b9c951] 2024-12-09T10:43:17,909 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412091fdfe00bd2c7424fadb91a0a11fcee79_92c0b293dd887d05ae6f705c47b9c951 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=92c0b293dd887d05ae6f705c47b9c951] 2024-12-09T10:43:17,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742370_1546 (size=12459) 2024-12-09T10:43:17,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742371_1547 (size=4469) 2024-12-09T10:43:17,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,939 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/B/07072ac26e9b4995aba7c749e9c85570 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/07072ac26e9b4995aba7c749e9c85570 2024-12-09T10:43:17,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,945 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 92c0b293dd887d05ae6f705c47b9c951/B of 92c0b293dd887d05ae6f705c47b9c951 into 07072ac26e9b4995aba7c749e9c85570(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:43:17,946 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 92c0b293dd887d05ae6f705c47b9c951: 2024-12-09T10:43:17,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,946 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., storeName=92c0b293dd887d05ae6f705c47b9c951/B, priority=13, startTime=1733740997890; duration=0sec 2024-12-09T10:43:17,946 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T10:43:17,946 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 92c0b293dd887d05ae6f705c47b9c951:B 2024-12-09T10:43:17,946 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T10:43:17,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,947 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T10:43:17,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,947 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1541): 92c0b293dd887d05ae6f705c47b9c951/C is initiating minor compaction (all files) 2024-12-09T10:43:17,947 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 92c0b293dd887d05ae6f705c47b9c951/C in TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:17,947 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/59f7b5186ffe4c2dad6b664d6e1dc9b8, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/faff8273cb8048ddbae1d87d05b35d96, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/03c22604e71f453fa94b830a8878ac51] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp, totalSize=35.5 K 2024-12-09T10:43:17,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,947 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 59f7b5186ffe4c2dad6b664d6e1dc9b8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1733740995092 2024-12-09T10:43:17,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,948 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting faff8273cb8048ddbae1d87d05b35d96, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1733740995101 2024-12-09T10:43:17,948 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 03c22604e71f453fa94b830a8878ac51, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1733740995857 2024-12-09T10:43:17,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,957 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 92c0b293dd887d05ae6f705c47b9c951#C#compaction#463 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:43:17,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,957 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/C/3e655c7f6a1a4ba9905fd9dc5f4c664c is 50, key is test_row_0/C:col10/1733740996972/Put/seqid=0 2024-12-09T10:43:17,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742372_1548 (size=12459) 2024-12-09T10:43:17,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,968 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/C/3e655c7f6a1a4ba9905fd9dc5f4c664c as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/3e655c7f6a1a4ba9905fd9dc5f4c664c 2024-12-09T10:43:17,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,974 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 92c0b293dd887d05ae6f705c47b9c951/C of 92c0b293dd887d05ae6f705c47b9c951 into 3e655c7f6a1a4ba9905fd9dc5f4c664c(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:43:17,974 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 92c0b293dd887d05ae6f705c47b9c951: 2024-12-09T10:43:17,974 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., storeName=92c0b293dd887d05ae6f705c47b9c951/C, priority=13, startTime=1733740997890; duration=0sec 2024-12-09T10:43:17,974 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:43:17,974 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 92c0b293dd887d05ae6f705c47b9c951:C 2024-12-09T10:43:17,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:17,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:18,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:18,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:18,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:18,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:18,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:18,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:18,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:18,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:18,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:18,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:18,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:18,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:18,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:18,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:18,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:18,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:18,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:18,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:18,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:18,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:18,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:18,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:18,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:18,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:18,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:18,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:18,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:18,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:18,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:18,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:18,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:18,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:18,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:18,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:18,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:18,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:18,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:18,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:18,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:18,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:18,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:18,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:18,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:18,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:18,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:18,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:18,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:18,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:18,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:18,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:18,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:18,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:18,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:18,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:18,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:18,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:18,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:18,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:18,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:18,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:18,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:18,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:18,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:18,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:18,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:18,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:18,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:18,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:18,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:18,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:18,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:18,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:18,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:18,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:18,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:18,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:18,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:18,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:18,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:18,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:18,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:18,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(8855): Flush requested on 92c0b293dd887d05ae6f705c47b9c951 2024-12-09T10:43:18,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:18,030 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 92c0b293dd887d05ae6f705c47b9c951 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-09T10:43:18,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:18,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 92c0b293dd887d05ae6f705c47b9c951, store=A 2024-12-09T10:43:18,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:43:18,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 92c0b293dd887d05ae6f705c47b9c951, store=B 2024-12-09T10:43:18,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:18,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:43:18,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 92c0b293dd887d05ae6f705c47b9c951, store=C 2024-12-09T10:43:18,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:43:18,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:18,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:18,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:18,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:18,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:18,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:18,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:18,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:18,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:18,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:18,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:18,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:18,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:18,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:18,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:18,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:18,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:18,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:18,037 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-09T10:43:18,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:18,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:18,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. as already flushing 2024-12-09T10:43:18,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:18,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:18,037 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:43:18,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:43:18,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:18,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:18,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:43:18,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:18,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:18,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:18,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:18,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:18,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:18,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:18,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:18,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:18,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:18,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:18,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:18,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:18,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:18,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:18,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:18,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:18,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:18,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:18,047 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412093e0945c50c9b4ded848e47559ce59289_92c0b293dd887d05ae6f705c47b9c951 is 50, key is test_row_1/A:col10/1733740998023/Put/seqid=0 2024-12-09T10:43:18,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-09T10:43:18,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742373_1549 (size=14794) 2024-12-09T10:43:18,056 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:18,060 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412093e0945c50c9b4ded848e47559ce59289_92c0b293dd887d05ae6f705c47b9c951 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412093e0945c50c9b4ded848e47559ce59289_92c0b293dd887d05ae6f705c47b9c951 2024-12-09T10:43:18,061 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/A/ca2de9012aa54cc489fcf74c58edc76f, store: [table=TestAcidGuarantees family=A region=92c0b293dd887d05ae6f705c47b9c951] 2024-12-09T10:43:18,062 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/A/ca2de9012aa54cc489fcf74c58edc76f is 175, key is test_row_1/A:col10/1733740998023/Put/seqid=0 2024-12-09T10:43:18,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742374_1550 (size=39745) 2024-12-09T10:43:18,104 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:18,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60502 deadline: 1733741058075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:18,106 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733740998065, max timestamp=1733740998074Num uniques 859; 2024-12-09T10:43:18,106 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733740998065, max timestamp=1733740998086Num uniques 859; 2024-12-09T10:43:18,106 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T10:43:18,106 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T10:43:18,106 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733740998065, max timestamp=1733740998086Num uniques 859; 2024-12-09T10:43:18,107 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T10:43:18,107 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:18,107 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:18,107 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:18,140 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:18,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60526 deadline: 1733741058134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:18,141 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:18,141 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:18,141 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:18,141 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:18,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60518 deadline: 1733741058136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:18,142 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:18,142 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:18,142 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:18,145 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:18,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60464 deadline: 1733741058141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:18,145 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:18,145 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:18,145 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:18,163 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:18,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60494 deadline: 1733741058159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:18,164 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:18,164 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:18,164 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:18,192 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-09T10:43:18,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:18,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. as already flushing 2024-12-09T10:43:18,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:18,192 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:43:18,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:43:18,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:43:18,213 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:18,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60502 deadline: 1733741058210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:18,214 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:18,214 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:18,214 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:18,251 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:18,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60526 deadline: 1733741058249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:18,252 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:18,252 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:18,252 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:18,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60518 deadline: 1733741058249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:18,252 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:18,252 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:18,252 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:18,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60464 deadline: 1733741058249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:18,252 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:18,252 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:18,253 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:18,253 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:18,253 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:18,340 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 92c0b293dd887d05ae6f705c47b9c951#A#compaction#461 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:43:18,340 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/A/c2ecf4bd12cd4867b4fea00bb3d76cb5 is 175, key is test_row_0/A:col10/1733740996972/Put/seqid=0 2024-12-09T10:43:18,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742375_1551 (size=31413) 2024-12-09T10:43:18,345 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-09T10:43:18,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:18,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. as already flushing 2024-12-09T10:43:18,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:18,346 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:43:18,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:43:18,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:43:18,424 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:18,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60502 deadline: 1733741058420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:18,425 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:18,425 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:18,425 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:18,465 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:18,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60526 deadline: 1733741058459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:18,465 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:18,465 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:18,466 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:18,466 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:18,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60518 deadline: 1733741058459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:18,466 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:18,466 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:18,466 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:18,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60464 deadline: 1733741058459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:18,466 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:18,466 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:18,466 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:18,466 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:18,471 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=172, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/A/ca2de9012aa54cc489fcf74c58edc76f 2024-12-09T10:43:18,477 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/B/d8c3546531234dd5a54dafc4d7bb2053 is 50, key is test_row_1/B:col10/1733740998023/Put/seqid=0 2024-12-09T10:43:18,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742376_1552 (size=9757) 2024-12-09T10:43:18,499 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-09T10:43:18,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:18,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. as already flushing 2024-12-09T10:43:18,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:18,499 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:43:18,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:43:18,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:43:18,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-09T10:43:18,652 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-09T10:43:18,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:18,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. as already flushing 2024-12-09T10:43:18,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:18,652 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:43:18,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:43:18,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:43:18,733 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:18,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60502 deadline: 1733741058729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:18,734 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:18,734 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:18,734 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:18,749 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/A/c2ecf4bd12cd4867b4fea00bb3d76cb5 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/c2ecf4bd12cd4867b4fea00bb3d76cb5 2024-12-09T10:43:18,757 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 92c0b293dd887d05ae6f705c47b9c951/A of 92c0b293dd887d05ae6f705c47b9c951 into c2ecf4bd12cd4867b4fea00bb3d76cb5(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:43:18,757 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 92c0b293dd887d05ae6f705c47b9c951: 2024-12-09T10:43:18,757 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., storeName=92c0b293dd887d05ae6f705c47b9c951/A, priority=13, startTime=1733740997887; duration=0sec 2024-12-09T10:43:18,757 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:43:18,757 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 92c0b293dd887d05ae6f705c47b9c951:A 2024-12-09T10:43:18,771 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:18,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60518 deadline: 1733741058769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:18,772 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:18,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60464 deadline: 1733741058770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:18,772 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:18,772 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:18,772 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:18,772 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:18,772 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:18,772 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:18,782 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:18,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60526 deadline: 1733741058780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:18,782 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:18,782 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:18,782 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:18,805 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-09T10:43:18,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:18,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. as already flushing 2024-12-09T10:43:18,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:18,806 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:43:18,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:43:18,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:43:18,881 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/B/d8c3546531234dd5a54dafc4d7bb2053 2024-12-09T10:43:18,888 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/C/68aeed3e213d4d7bb39ddd3c053ba113 is 50, key is test_row_1/C:col10/1733740998023/Put/seqid=0 2024-12-09T10:43:18,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742377_1553 (size=9757) 2024-12-09T10:43:18,905 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/C/68aeed3e213d4d7bb39ddd3c053ba113 2024-12-09T10:43:18,912 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/A/ca2de9012aa54cc489fcf74c58edc76f as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/ca2de9012aa54cc489fcf74c58edc76f 2024-12-09T10:43:18,916 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/ca2de9012aa54cc489fcf74c58edc76f, entries=200, sequenceid=172, filesize=38.8 K 2024-12-09T10:43:18,917 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/B/d8c3546531234dd5a54dafc4d7bb2053 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/d8c3546531234dd5a54dafc4d7bb2053 2024-12-09T10:43:18,924 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/d8c3546531234dd5a54dafc4d7bb2053, entries=100, sequenceid=172, filesize=9.5 K 2024-12-09T10:43:18,925 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/C/68aeed3e213d4d7bb39ddd3c053ba113 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/68aeed3e213d4d7bb39ddd3c053ba113 2024-12-09T10:43:18,928 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/68aeed3e213d4d7bb39ddd3c053ba113, entries=100, sequenceid=172, filesize=9.5 K 2024-12-09T10:43:18,930 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 92c0b293dd887d05ae6f705c47b9c951 in 899ms, sequenceid=172, compaction requested=false 2024-12-09T10:43:18,930 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 92c0b293dd887d05ae6f705c47b9c951: 2024-12-09T10:43:18,959 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-09T10:43:18,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:18,959 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing 92c0b293dd887d05ae6f705c47b9c951 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-12-09T10:43:18,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 92c0b293dd887d05ae6f705c47b9c951, store=A 2024-12-09T10:43:18,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T10:43:18,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 92c0b293dd887d05ae6f705c47b9c951, store=B 2024-12-09T10:43:18,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T10:43:18,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 92c0b293dd887d05ae6f705c47b9c951, store=C 2024-12-09T10:43:18,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T10:43:18,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209ca2bdb79413d4d80a61e055ed74a01e8_92c0b293dd887d05ae6f705c47b9c951 is 50, key is test_row_0/A:col10/1733740998139/Put/seqid=0 2024-12-09T10:43:19,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742378_1554 (size=12304) 2024-12-09T10:43:19,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:19,029 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209ca2bdb79413d4d80a61e055ed74a01e8_92c0b293dd887d05ae6f705c47b9c951 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209ca2bdb79413d4d80a61e055ed74a01e8_92c0b293dd887d05ae6f705c47b9c951 2024-12-09T10:43:19,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/A/6d442efaf1784ee98612b5b5d7710035, store: [table=TestAcidGuarantees family=A region=92c0b293dd887d05ae6f705c47b9c951] 2024-12-09T10:43:19,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/A/6d442efaf1784ee98612b5b5d7710035 is 175, key is test_row_0/A:col10/1733740998139/Put/seqid=0 2024-12-09T10:43:19,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742379_1555 (size=31105) 2024-12-09T10:43:19,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(8855): Flush requested on 92c0b293dd887d05ae6f705c47b9c951 2024-12-09T10:43:19,183 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. as already flushing 2024-12-09T10:43:19,225 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:19,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60494 deadline: 1733741059219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:19,226 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:19,226 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:19,226 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:19,243 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:19,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60502 deadline: 1733741059240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:19,244 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:19,244 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:19,244 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:19,282 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:19,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60518 deadline: 1733741059279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:19,282 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:19,282 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:19,283 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:19,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60464 deadline: 1733741059279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:19,283 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:19,283 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:19,283 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:19,283 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:19,293 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:19,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60526 deadline: 1733741059289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:19,294 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:19,294 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:19,294 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:19,331 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:19,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60494 deadline: 1733741059330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:19,332 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:19,332 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:19,332 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:19,448 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=210, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/A/6d442efaf1784ee98612b5b5d7710035 2024-12-09T10:43:19,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/B/d45462eac83e493f99272b1d5467e24b is 50, key is test_row_0/B:col10/1733740998139/Put/seqid=0 2024-12-09T10:43:19,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742380_1556 (size=12151) 2024-12-09T10:43:19,542 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:19,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60494 deadline: 1733741059539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:19,542 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:19,542 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:19,542 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:19,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-09T10:43:19,851 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:19,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60494 deadline: 1733741059850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:19,852 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:19,852 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:19,852 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:19,858 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/B/d45462eac83e493f99272b1d5467e24b 2024-12-09T10:43:19,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/C/da7a8dd8545b45b48c7e1bb108ba80d2 is 50, key is test_row_0/C:col10/1733740998139/Put/seqid=0 2024-12-09T10:43:19,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742381_1557 (size=12151) 2024-12-09T10:43:20,251 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:20,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60502 deadline: 1733741060249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:20,251 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 7ea43b39fc24,36101,1733740827678 for 'test_row_0' in 92c0b293dd887d05ae6f705c47b9c951 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T10:43:20,252 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:20,252 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:20,252 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:20,270 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/C/da7a8dd8545b45b48c7e1bb108ba80d2 2024-12-09T10:43:20,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/A/6d442efaf1784ee98612b5b5d7710035 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/6d442efaf1784ee98612b5b5d7710035 2024-12-09T10:43:20,277 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/6d442efaf1784ee98612b5b5d7710035, entries=150, sequenceid=210, filesize=30.4 K 2024-12-09T10:43:20,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/B/d45462eac83e493f99272b1d5467e24b as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/d45462eac83e493f99272b1d5467e24b 2024-12-09T10:43:20,281 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/d45462eac83e493f99272b1d5467e24b, entries=150, sequenceid=210, filesize=11.9 K 2024-12-09T10:43:20,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/C/da7a8dd8545b45b48c7e1bb108ba80d2 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/da7a8dd8545b45b48c7e1bb108ba80d2 2024-12-09T10:43:20,285 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/da7a8dd8545b45b48c7e1bb108ba80d2, entries=150, sequenceid=210, filesize=11.9 K 2024-12-09T10:43:20,286 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 92c0b293dd887d05ae6f705c47b9c951 in 1327ms, sequenceid=210, compaction requested=true 2024-12-09T10:43:20,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for 92c0b293dd887d05ae6f705c47b9c951: 2024-12-09T10:43:20,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:20,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-12-09T10:43:20,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-12-09T10:43:20,289 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-12-09T10:43:20,289 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8620 sec 2024-12-09T10:43:20,291 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 2.8690 sec 2024-12-09T10:43:20,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(8855): Flush requested on 92c0b293dd887d05ae6f705c47b9c951 2024-12-09T10:43:20,292 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 92c0b293dd887d05ae6f705c47b9c951 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-09T10:43:20,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 92c0b293dd887d05ae6f705c47b9c951, store=A 2024-12-09T10:43:20,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:43:20,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 92c0b293dd887d05ae6f705c47b9c951, store=B 2024-12-09T10:43:20,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:43:20,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 92c0b293dd887d05ae6f705c47b9c951, store=C 2024-12-09T10:43:20,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:43:20,299 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209d8f3af9deda34709a0cfa20ec084a876_92c0b293dd887d05ae6f705c47b9c951 is 50, key is test_row_0/A:col10/1733741000291/Put/seqid=0 2024-12-09T10:43:20,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742382_1558 (size=14794) 2024-12-09T10:43:20,340 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733741000327, max timestamp=1733741000337Num uniques 859; 2024-12-09T10:43:20,340 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T10:43:20,341 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733741000327, max timestamp=1733741000337Num uniques 859; 2024-12-09T10:43:20,341 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T10:43:20,342 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733741000327, max timestamp=1733741000337Num uniques 859; 2024-12-09T10:43:20,342 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T10:43:20,381 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733741000358, max timestamp=1733741000376Num uniques 1018; 2024-12-09T10:43:20,381 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733741000358, max timestamp=1733741000376Num uniques 1018; 2024-12-09T10:43:20,381 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-09T10:43:20,381 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733741000358, max timestamp=1733741000376Num uniques 1018; 2024-12-09T10:43:20,381 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-09T10:43:20,381 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-09T10:43:20,410 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:20,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60494 deadline: 1733741060403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:20,411 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:20,411 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:20,411 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:20,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60464 deadline: 1733741060404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:20,411 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:20,411 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:20,411 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:20,411 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:20,413 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:20,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60526 deadline: 1733741060406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:20,413 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:20,413 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:20,413 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:20,413 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:20,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60518 deadline: 1733741060407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:20,413 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:20,413 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:20,413 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:20,522 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:20,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60494 deadline: 1733741060519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:20,523 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:20,523 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:20,523 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:20,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60464 deadline: 1733741060519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:20,523 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:20,523 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:20,523 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:20,523 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:20,523 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:20,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60526 deadline: 1733741060519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:20,523 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:20,523 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:20,523 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:20,523 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:20,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60518 deadline: 1733741060519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:20,524 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:20,524 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:20,524 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:20,711 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:20,715 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209d8f3af9deda34709a0cfa20ec084a876_92c0b293dd887d05ae6f705c47b9c951 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209d8f3af9deda34709a0cfa20ec084a876_92c0b293dd887d05ae6f705c47b9c951 2024-12-09T10:43:20,715 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/A/f30ff041cda942458022fbb6ed798a8c, store: [table=TestAcidGuarantees family=A region=92c0b293dd887d05ae6f705c47b9c951] 2024-12-09T10:43:20,716 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/A/f30ff041cda942458022fbb6ed798a8c is 175, key is test_row_0/A:col10/1733741000291/Put/seqid=0 2024-12-09T10:43:20,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742383_1559 (size=39749) 2024-12-09T10:43:20,732 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:20,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60494 deadline: 1733741060729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:20,732 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:20,732 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:20,733 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:20,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60464 deadline: 1733741060729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:20,733 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:20,733 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:20,733 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:20,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60526 deadline: 1733741060729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:20,733 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:20,733 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:20,733 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:20,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60518 deadline: 1733741060729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:20,733 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:20,733 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:20,733 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:20,733 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:20,733 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:20,733 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:21,044 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:21,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60494 deadline: 1733741061039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:21,044 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:21,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60464 deadline: 1733741061039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:21,044 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:21,044 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:21,044 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:21,044 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:21,045 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:21,045 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:21,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60518 deadline: 1733741061039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:21,045 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:21,045 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:21,045 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:21,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60526 deadline: 1733741061040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:21,045 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:21,045 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:21,045 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:21,045 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:21,045 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:21,119 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=221, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/A/f30ff041cda942458022fbb6ed798a8c 2024-12-09T10:43:21,126 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/B/dc507977e37e4a629ad46e4272127702 is 50, key is test_row_0/B:col10/1733741000291/Put/seqid=0 2024-12-09T10:43:21,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742384_1560 (size=12151) 2024-12-09T10:43:21,530 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/B/dc507977e37e4a629ad46e4272127702 2024-12-09T10:43:21,537 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/C/d513904c790f4e4ab13f9c2682d147c6 is 50, key is test_row_0/C:col10/1733741000291/Put/seqid=0 2024-12-09T10:43:21,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742385_1561 (size=12151) 2024-12-09T10:43:21,552 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:21,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60494 deadline: 1733741061549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:21,553 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:21,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60464 deadline: 1733741061549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:21,553 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:21,553 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:21,553 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:21,553 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:21,553 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:21,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60518 deadline: 1733741061549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:21,553 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:21,553 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:21,553 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:21,553 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:21,553 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:21,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:21,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60526 deadline: 1733741061560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:21,564 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:21,564 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:21,564 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:21,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-09T10:43:21,579 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T10:43:21,580 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-09T10:43:21,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-12-09T10:43:21,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-09T10:43:21,582 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T10:43:21,582 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T10:43:21,582 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T10:43:21,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-09T10:43:21,735 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-09T10:43:21,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:21,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. as already flushing 2024-12-09T10:43:21,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:21,735 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:43:21,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:43:21,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:43:21,888 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-09T10:43:21,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:21,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. as already flushing 2024-12-09T10:43:21,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:21,888 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:43:21,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:43:21,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:43:21,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-09T10:43:21,940 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/C/d513904c790f4e4ab13f9c2682d147c6 2024-12-09T10:43:21,944 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/A/f30ff041cda942458022fbb6ed798a8c as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/f30ff041cda942458022fbb6ed798a8c 2024-12-09T10:43:21,947 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/f30ff041cda942458022fbb6ed798a8c, entries=200, sequenceid=221, filesize=38.8 K 2024-12-09T10:43:21,949 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/B/dc507977e37e4a629ad46e4272127702 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/dc507977e37e4a629ad46e4272127702 2024-12-09T10:43:21,952 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/dc507977e37e4a629ad46e4272127702, entries=150, sequenceid=221, filesize=11.9 K 2024-12-09T10:43:21,953 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/C/d513904c790f4e4ab13f9c2682d147c6 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/d513904c790f4e4ab13f9c2682d147c6 2024-12-09T10:43:21,957 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/d513904c790f4e4ab13f9c2682d147c6, entries=150, sequenceid=221, filesize=11.9 K 2024-12-09T10:43:21,958 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 92c0b293dd887d05ae6f705c47b9c951 in 1665ms, sequenceid=221, compaction requested=true 2024-12-09T10:43:21,958 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 92c0b293dd887d05ae6f705c47b9c951: 2024-12-09T10:43:21,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 92c0b293dd887d05ae6f705c47b9c951:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T10:43:21,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:43:21,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 92c0b293dd887d05ae6f705c47b9c951:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T10:43:21,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:43:21,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 92c0b293dd887d05ae6f705c47b9c951:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T10:43:21,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T10:43:21,958 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T10:43:21,958 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T10:43:21,959 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46518 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T10:43:21,959 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1541): 92c0b293dd887d05ae6f705c47b9c951/B is initiating minor compaction (all files) 2024-12-09T10:43:21,959 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 92c0b293dd887d05ae6f705c47b9c951/B in TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:21,959 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/07072ac26e9b4995aba7c749e9c85570, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/d8c3546531234dd5a54dafc4d7bb2053, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/d45462eac83e493f99272b1d5467e24b, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/dc507977e37e4a629ad46e4272127702] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp, totalSize=45.4 K 2024-12-09T10:43:21,960 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142012 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T10:43:21,960 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1541): 92c0b293dd887d05ae6f705c47b9c951/A is initiating minor compaction (all files) 2024-12-09T10:43:21,960 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 92c0b293dd887d05ae6f705c47b9c951/A in TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:21,960 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/c2ecf4bd12cd4867b4fea00bb3d76cb5, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/ca2de9012aa54cc489fcf74c58edc76f, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/6d442efaf1784ee98612b5b5d7710035, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/f30ff041cda942458022fbb6ed798a8c] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp, totalSize=138.7 K 2024-12-09T10:43:21,960 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:21,960 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. files: [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/c2ecf4bd12cd4867b4fea00bb3d76cb5, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/ca2de9012aa54cc489fcf74c58edc76f, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/6d442efaf1784ee98612b5b5d7710035, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/f30ff041cda942458022fbb6ed798a8c] 2024-12-09T10:43:21,960 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 07072ac26e9b4995aba7c749e9c85570, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1733740995857 2024-12-09T10:43:21,961 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting c2ecf4bd12cd4867b4fea00bb3d76cb5, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1733740995857 2024-12-09T10:43:21,961 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting ca2de9012aa54cc489fcf74c58edc76f, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1733740996977 2024-12-09T10:43:21,961 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting d8c3546531234dd5a54dafc4d7bb2053, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1733740998023 2024-12-09T10:43:21,961 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting d45462eac83e493f99272b1d5467e24b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1733740998118 2024-12-09T10:43:21,961 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6d442efaf1784ee98612b5b5d7710035, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1733740998118 2024-12-09T10:43:21,962 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting dc507977e37e4a629ad46e4272127702, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1733740999203 2024-12-09T10:43:21,962 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting f30ff041cda942458022fbb6ed798a8c, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1733740999203 2024-12-09T10:43:21,971 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 92c0b293dd887d05ae6f705c47b9c951#B#compaction#473 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:43:21,972 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/B/b434797b8ca5484eaea5cda03c94e994 is 50, key is test_row_0/B:col10/1733741000291/Put/seqid=0 2024-12-09T10:43:21,974 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=92c0b293dd887d05ae6f705c47b9c951] 2024-12-09T10:43:21,976 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241209ab40e3206bd54e1d99931e8d564d36aa_92c0b293dd887d05ae6f705c47b9c951 store=[table=TestAcidGuarantees family=A region=92c0b293dd887d05ae6f705c47b9c951] 2024-12-09T10:43:21,978 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241209ab40e3206bd54e1d99931e8d564d36aa_92c0b293dd887d05ae6f705c47b9c951, store=[table=TestAcidGuarantees family=A region=92c0b293dd887d05ae6f705c47b9c951] 2024-12-09T10:43:21,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742386_1562 (size=12595) 2024-12-09T10:43:21,978 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209ab40e3206bd54e1d99931e8d564d36aa_92c0b293dd887d05ae6f705c47b9c951 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=92c0b293dd887d05ae6f705c47b9c951] 2024-12-09T10:43:21,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742387_1563 (size=4469) 2024-12-09T10:43:21,983 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 92c0b293dd887d05ae6f705c47b9c951#A#compaction#474 average throughput is 2.71 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:43:21,984 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/A/8778a840fb5f4e61ab802ecb2897f9f6 is 175, key is test_row_0/A:col10/1733741000291/Put/seqid=0 2024-12-09T10:43:21,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742388_1564 (size=31549) 2024-12-09T10:43:21,991 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/A/8778a840fb5f4e61ab802ecb2897f9f6 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/8778a840fb5f4e61ab802ecb2897f9f6 2024-12-09T10:43:21,995 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 92c0b293dd887d05ae6f705c47b9c951/A of 92c0b293dd887d05ae6f705c47b9c951 into 8778a840fb5f4e61ab802ecb2897f9f6(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:43:21,995 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 92c0b293dd887d05ae6f705c47b9c951: 2024-12-09T10:43:21,996 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., storeName=92c0b293dd887d05ae6f705c47b9c951/A, priority=12, startTime=1733741001958; duration=0sec 2024-12-09T10:43:21,996 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T10:43:21,996 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 92c0b293dd887d05ae6f705c47b9c951:A 2024-12-09T10:43:21,996 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T10:43:21,997 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46518 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T10:43:21,997 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1541): 92c0b293dd887d05ae6f705c47b9c951/C is initiating minor compaction (all files) 2024-12-09T10:43:21,997 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 92c0b293dd887d05ae6f705c47b9c951/C in TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:21,997 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/3e655c7f6a1a4ba9905fd9dc5f4c664c, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/68aeed3e213d4d7bb39ddd3c053ba113, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/da7a8dd8545b45b48c7e1bb108ba80d2, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/d513904c790f4e4ab13f9c2682d147c6] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp, totalSize=45.4 K 2024-12-09T10:43:21,997 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3e655c7f6a1a4ba9905fd9dc5f4c664c, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1733740995857 2024-12-09T10:43:21,998 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 68aeed3e213d4d7bb39ddd3c053ba113, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1733740998023 2024-12-09T10:43:21,998 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting da7a8dd8545b45b48c7e1bb108ba80d2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1733740998118 2024-12-09T10:43:21,998 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting d513904c790f4e4ab13f9c2682d147c6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1733740999203 2024-12-09T10:43:22,007 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 92c0b293dd887d05ae6f705c47b9c951#C#compaction#475 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:43:22,008 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/C/86360001169a45a78d38183010f9cc23 is 50, key is test_row_0/C:col10/1733741000291/Put/seqid=0 2024-12-09T10:43:22,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742389_1565 (size=12595) 2024-12-09T10:43:22,041 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-09T10:43:22,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:22,041 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing 92c0b293dd887d05ae6f705c47b9c951 3/3 column families, dataSize=127.47 KB heapSize=359.53 KB 2024-12-09T10:43:22,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 92c0b293dd887d05ae6f705c47b9c951, store=A 2024-12-09T10:43:22,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T10:43:22,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 92c0b293dd887d05ae6f705c47b9c951, store=B 2024-12-09T10:43:22,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T10:43:22,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 92c0b293dd887d05ae6f705c47b9c951, store=C 2024-12-09T10:43:22,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T10:43:22,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412095bcdf44aa36a451aa7ffefff561b0286_92c0b293dd887d05ae6f705c47b9c951 is 50, key is test_row_0/A:col10/1733741000405/Put/seqid=0 2024-12-09T10:43:22,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742390_1566 (size=12454) 2024-12-09T10:43:22,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-09T10:43:22,274 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. as already flushing 2024-12-09T10:43:22,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(8855): Flush requested on 92c0b293dd887d05ae6f705c47b9c951 2024-12-09T10:43:22,321 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:22,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60502 deadline: 1733741062316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:22,322 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:22,322 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:22,322 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:22,383 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/B/b434797b8ca5484eaea5cda03c94e994 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/b434797b8ca5484eaea5cda03c94e994 2024-12-09T10:43:22,387 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 92c0b293dd887d05ae6f705c47b9c951/B of 92c0b293dd887d05ae6f705c47b9c951 into b434797b8ca5484eaea5cda03c94e994(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:43:22,387 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 92c0b293dd887d05ae6f705c47b9c951: 2024-12-09T10:43:22,387 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., storeName=92c0b293dd887d05ae6f705c47b9c951/B, priority=12, startTime=1733741001958; duration=0sec 2024-12-09T10:43:22,387 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:43:22,387 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 92c0b293dd887d05ae6f705c47b9c951:B 2024-12-09T10:43:22,420 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/C/86360001169a45a78d38183010f9cc23 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/86360001169a45a78d38183010f9cc23 2024-12-09T10:43:22,424 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 92c0b293dd887d05ae6f705c47b9c951/C of 92c0b293dd887d05ae6f705c47b9c951 into 86360001169a45a78d38183010f9cc23(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:43:22,424 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 92c0b293dd887d05ae6f705c47b9c951: 2024-12-09T10:43:22,424 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., storeName=92c0b293dd887d05ae6f705c47b9c951/C, priority=12, startTime=1733741001958; duration=0sec 2024-12-09T10:43:22,425 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:43:22,425 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 92c0b293dd887d05ae6f705c47b9c951:C 2024-12-09T10:43:22,431 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:22,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60502 deadline: 1733741062429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:22,432 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:22,432 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:22,432 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:22,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:22,459 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412095bcdf44aa36a451aa7ffefff561b0286_92c0b293dd887d05ae6f705c47b9c951 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412095bcdf44aa36a451aa7ffefff561b0286_92c0b293dd887d05ae6f705c47b9c951 2024-12-09T10:43:22,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/A/98c0729217644a6d86c1b14e412e12db, store: [table=TestAcidGuarantees family=A region=92c0b293dd887d05ae6f705c47b9c951] 2024-12-09T10:43:22,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/A/98c0729217644a6d86c1b14e412e12db is 175, key is test_row_0/A:col10/1733741000405/Put/seqid=0 2024-12-09T10:43:22,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742391_1567 (size=31255) 2024-12-09T10:43:22,563 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:22,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60494 deadline: 1733741062559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:22,563 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:22,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60518 deadline: 1733741062559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:22,563 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 7ea43b39fc24,36101,1733740827678 for 'test_row_1' in 92c0b293dd887d05ae6f705c47b9c951 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2160 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T10:43:22,564 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 7ea43b39fc24,36101,1733740827678 for 'test_row_2' in 92c0b293dd887d05ae6f705c47b9c951 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2157 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T10:43:22,564 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:22,564 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:22,564 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:22,564 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:22,564 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:22,564 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:22,573 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:22,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60464 deadline: 1733741062570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:22,573 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 7ea43b39fc24,36101,1733740827678 for 'test_row_2' in 92c0b293dd887d05ae6f705c47b9c951 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T10:43:22,574 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:22,574 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:22,574 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:22,583 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:22,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60526 deadline: 1733741062580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:22,583 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 7ea43b39fc24,36101,1733740827678 for 'test_row_0' in 92c0b293dd887d05ae6f705c47b9c951 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T10:43:22,584 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:22,584 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:22,584 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:22,642 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:22,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60502 deadline: 1733741062639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:22,643 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:22,643 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:22,643 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:22,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-09T10:43:22,864 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=274, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/A/98c0729217644a6d86c1b14e412e12db 2024-12-09T10:43:22,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/B/b545338a8d4f4120aede397c7c1e031c is 50, key is test_row_0/B:col10/1733741000405/Put/seqid=0 2024-12-09T10:43:22,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742392_1568 (size=12301) 2024-12-09T10:43:22,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:22,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60502 deadline: 1733741062949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:22,952 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:22,952 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:22,952 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:23,275 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/B/b545338a8d4f4120aede397c7c1e031c 2024-12-09T10:43:23,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/C/079c6231155d4a169d394c30f629feac is 50, key is test_row_0/C:col10/1733741000405/Put/seqid=0 2024-12-09T10:43:23,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742393_1569 (size=12301) 2024-12-09T10:43:23,285 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/C/079c6231155d4a169d394c30f629feac 2024-12-09T10:43:23,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/A/98c0729217644a6d86c1b14e412e12db as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/98c0729217644a6d86c1b14e412e12db 2024-12-09T10:43:23,291 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/98c0729217644a6d86c1b14e412e12db, entries=150, sequenceid=274, filesize=30.5 K 2024-12-09T10:43:23,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/B/b545338a8d4f4120aede397c7c1e031c as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/b545338a8d4f4120aede397c7c1e031c 2024-12-09T10:43:23,295 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/b545338a8d4f4120aede397c7c1e031c, entries=150, sequenceid=274, filesize=12.0 K 2024-12-09T10:43:23,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/C/079c6231155d4a169d394c30f629feac as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/079c6231155d4a169d394c30f629feac 2024-12-09T10:43:23,300 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/079c6231155d4a169d394c30f629feac, entries=150, sequenceid=274, filesize=12.0 K 2024-12-09T10:43:23,301 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~359.48 KB/368112, currentSize=53.67 KB/54960 for 92c0b293dd887d05ae6f705c47b9c951 in 1259ms, sequenceid=274, compaction requested=false 2024-12-09T10:43:23,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for 92c0b293dd887d05ae6f705c47b9c951: 2024-12-09T10:43:23,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:23,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-12-09T10:43:23,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-12-09T10:43:23,305 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-12-09T10:43:23,305 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7200 sec 2024-12-09T10:43:23,307 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 1.7250 sec 2024-12-09T10:43:23,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(8855): Flush requested on 92c0b293dd887d05ae6f705c47b9c951 2024-12-09T10:43:23,462 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 92c0b293dd887d05ae6f705c47b9c951 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-09T10:43:23,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 92c0b293dd887d05ae6f705c47b9c951, store=A 2024-12-09T10:43:23,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:43:23,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 92c0b293dd887d05ae6f705c47b9c951, store=B 2024-12-09T10:43:23,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:43:23,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 92c0b293dd887d05ae6f705c47b9c951, store=C 2024-12-09T10:43:23,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:43:23,468 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209f2b0d84b48a944bd8ab4eb66a9ea8c70_92c0b293dd887d05ae6f705c47b9c951 is 50, key is test_row_0/A:col10/1733741003460/Put/seqid=0 2024-12-09T10:43:23,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742394_1570 (size=14994) 2024-12-09T10:43:23,545 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:23,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60502 deadline: 1733741063541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:23,546 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:23,546 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:23,546 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:23,653 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:23,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60502 deadline: 1733741063650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:23,653 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:23,653 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:23,653 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:23,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-09T10:43:23,730 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T10:43:23,730 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-09T10:43:23,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-12-09T10:43:23,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-09T10:43:23,732 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T10:43:23,732 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T10:43:23,733 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T10:43:23,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-09T10:43:23,860 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:23,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60502 deadline: 1733741063860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:23,861 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:23,861 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:23,861 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:23,872 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:23,876 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209f2b0d84b48a944bd8ab4eb66a9ea8c70_92c0b293dd887d05ae6f705c47b9c951 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209f2b0d84b48a944bd8ab4eb66a9ea8c70_92c0b293dd887d05ae6f705c47b9c951 2024-12-09T10:43:23,876 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/A/5b71d7f4ee1c4788ba845606e0e3c9d2, store: [table=TestAcidGuarantees family=A region=92c0b293dd887d05ae6f705c47b9c951] 2024-12-09T10:43:23,877 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/A/5b71d7f4ee1c4788ba845606e0e3c9d2 is 175, key is test_row_0/A:col10/1733741003460/Put/seqid=0 2024-12-09T10:43:23,885 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-09T10:43:23,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:23,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. as already flushing 2024-12-09T10:43:23,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:23,885 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:43:23,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:43:23,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:43:23,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742395_1571 (size=39949) 2024-12-09T10:43:24,038 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-09T10:43:24,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:24,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. as already flushing 2024-12-09T10:43:24,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:24,038 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:43:24,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:43:24,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:43:24,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-09T10:43:24,171 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:24,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60502 deadline: 1733741064169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:24,172 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:24,172 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:24,172 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:24,191 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-09T10:43:24,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:24,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. as already flushing 2024-12-09T10:43:24,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:24,191 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:43:24,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:43:24,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:43:24,287 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=288, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/A/5b71d7f4ee1c4788ba845606e0e3c9d2 2024-12-09T10:43:24,293 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/B/20843e37ceab4c24bbf55d56416a1859 is 50, key is test_row_0/B:col10/1733741003460/Put/seqid=0 2024-12-09T10:43:24,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742396_1572 (size=12301) 2024-12-09T10:43:24,297 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/B/20843e37ceab4c24bbf55d56416a1859 2024-12-09T10:43:24,303 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/C/4ea5d3cd381f4031a25bdd6452226e1b is 50, key is test_row_0/C:col10/1733741003460/Put/seqid=0 2024-12-09T10:43:24,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742397_1573 (size=12301) 2024-12-09T10:43:24,343 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-09T10:43:24,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:24,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. as already flushing 2024-12-09T10:43:24,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:24,344 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:43:24,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:43:24,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:43:24,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-09T10:43:24,496 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-09T10:43:24,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:24,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. as already flushing 2024-12-09T10:43:24,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:24,497 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:43:24,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:43:24,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:43:24,573 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:24,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60494 deadline: 1733741064569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:24,573 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 7ea43b39fc24,36101,1733740827678 for 'test_row_1' in 92c0b293dd887d05ae6f705c47b9c951 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T10:43:24,574 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:24,574 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:24,574 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:24,583 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:24,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60518 deadline: 1733741064580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:24,583 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 7ea43b39fc24,36101,1733740827678 for 'test_row_2' in 92c0b293dd887d05ae6f705c47b9c951 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T10:43:24,584 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:24,584 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:24,584 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:24,592 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:24,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60464 deadline: 1733741064589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:24,592 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 7ea43b39fc24,36101,1733740827678 for 'test_row_2' in 92c0b293dd887d05ae6f705c47b9c951 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T10:43:24,592 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:24,593 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:24,593 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:24,601 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:24,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60526 deadline: 1733741064599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:24,601 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 7ea43b39fc24,36101,1733740827678 for 'test_row_0' in 92c0b293dd887d05ae6f705c47b9c951 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T10:43:24,602 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:24,602 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:24,602 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:24,655 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-09T10:43:24,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:24,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. as already flushing 2024-12-09T10:43:24,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:24,655 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:43:24,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:43:24,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:43:24,683 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:24,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60502 deadline: 1733741064679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:24,684 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:24,684 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:24,684 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:24,707 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/C/4ea5d3cd381f4031a25bdd6452226e1b 2024-12-09T10:43:24,711 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/A/5b71d7f4ee1c4788ba845606e0e3c9d2 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/5b71d7f4ee1c4788ba845606e0e3c9d2 2024-12-09T10:43:24,714 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/5b71d7f4ee1c4788ba845606e0e3c9d2, entries=200, sequenceid=288, filesize=39.0 K 2024-12-09T10:43:24,715 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/B/20843e37ceab4c24bbf55d56416a1859 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/20843e37ceab4c24bbf55d56416a1859 2024-12-09T10:43:24,718 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/20843e37ceab4c24bbf55d56416a1859, entries=150, sequenceid=288, filesize=12.0 K 2024-12-09T10:43:24,718 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/C/4ea5d3cd381f4031a25bdd6452226e1b as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/4ea5d3cd381f4031a25bdd6452226e1b 2024-12-09T10:43:24,722 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/4ea5d3cd381f4031a25bdd6452226e1b, entries=150, sequenceid=288, filesize=12.0 K 2024-12-09T10:43:24,722 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 92c0b293dd887d05ae6f705c47b9c951 in 1260ms, sequenceid=288, compaction requested=true 2024-12-09T10:43:24,722 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 92c0b293dd887d05ae6f705c47b9c951: 2024-12-09T10:43:24,722 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 92c0b293dd887d05ae6f705c47b9c951:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T10:43:24,722 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:43:24,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 92c0b293dd887d05ae6f705c47b9c951:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T10:43:24,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:43:24,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 92c0b293dd887d05ae6f705c47b9c951:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T10:43:24,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T10:43:24,723 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T10:43:24,723 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T10:43:24,723 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102753 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T10:43:24,723 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37197 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T10:43:24,724 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1541): 92c0b293dd887d05ae6f705c47b9c951/A is initiating minor compaction (all files) 2024-12-09T10:43:24,724 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1541): 92c0b293dd887d05ae6f705c47b9c951/B is initiating minor compaction (all files) 2024-12-09T10:43:24,724 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 92c0b293dd887d05ae6f705c47b9c951/A in TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:24,724 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/8778a840fb5f4e61ab802ecb2897f9f6, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/98c0729217644a6d86c1b14e412e12db, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/5b71d7f4ee1c4788ba845606e0e3c9d2] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp, totalSize=100.3 K 2024-12-09T10:43:24,724 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 92c0b293dd887d05ae6f705c47b9c951/B in TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:24,724 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:24,724 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. files: [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/8778a840fb5f4e61ab802ecb2897f9f6, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/98c0729217644a6d86c1b14e412e12db, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/5b71d7f4ee1c4788ba845606e0e3c9d2] 2024-12-09T10:43:24,724 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/b434797b8ca5484eaea5cda03c94e994, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/b545338a8d4f4120aede397c7c1e031c, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/20843e37ceab4c24bbf55d56416a1859] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp, totalSize=36.3 K 2024-12-09T10:43:24,724 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8778a840fb5f4e61ab802ecb2897f9f6, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1733740999203 2024-12-09T10:43:24,724 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting b434797b8ca5484eaea5cda03c94e994, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1733740999203 2024-12-09T10:43:24,725 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting b545338a8d4f4120aede397c7c1e031c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1733741000403 2024-12-09T10:43:24,725 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 98c0729217644a6d86c1b14e412e12db, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1733741000403 2024-12-09T10:43:24,726 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 20843e37ceab4c24bbf55d56416a1859, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1733741002308 2024-12-09T10:43:24,726 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b71d7f4ee1c4788ba845606e0e3c9d2, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1733741002301 2024-12-09T10:43:24,731 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=92c0b293dd887d05ae6f705c47b9c951] 2024-12-09T10:43:24,732 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 92c0b293dd887d05ae6f705c47b9c951#B#compaction#482 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:43:24,733 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/B/a84f584c5da84ce4b9a391cf1945842e is 50, key is test_row_0/B:col10/1733741003460/Put/seqid=0 2024-12-09T10:43:24,734 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412092ac422dec981408f8a3945f7d2360c1e_92c0b293dd887d05ae6f705c47b9c951 store=[table=TestAcidGuarantees family=A region=92c0b293dd887d05ae6f705c47b9c951] 2024-12-09T10:43:24,736 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412092ac422dec981408f8a3945f7d2360c1e_92c0b293dd887d05ae6f705c47b9c951, store=[table=TestAcidGuarantees family=A region=92c0b293dd887d05ae6f705c47b9c951] 2024-12-09T10:43:24,736 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412092ac422dec981408f8a3945f7d2360c1e_92c0b293dd887d05ae6f705c47b9c951 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=92c0b293dd887d05ae6f705c47b9c951] 2024-12-09T10:43:24,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742398_1574 (size=12847) 2024-12-09T10:43:24,748 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/B/a84f584c5da84ce4b9a391cf1945842e as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/a84f584c5da84ce4b9a391cf1945842e 2024-12-09T10:43:24,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742399_1575 (size=4469) 2024-12-09T10:43:24,753 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 92c0b293dd887d05ae6f705c47b9c951/B of 92c0b293dd887d05ae6f705c47b9c951 into a84f584c5da84ce4b9a391cf1945842e(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:43:24,753 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 92c0b293dd887d05ae6f705c47b9c951: 2024-12-09T10:43:24,753 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., storeName=92c0b293dd887d05ae6f705c47b9c951/B, priority=13, startTime=1733741004723; duration=0sec 2024-12-09T10:43:24,754 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T10:43:24,754 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 92c0b293dd887d05ae6f705c47b9c951:B 2024-12-09T10:43:24,754 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T10:43:24,755 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37197 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T10:43:24,755 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1541): 92c0b293dd887d05ae6f705c47b9c951/C is initiating minor compaction (all files) 2024-12-09T10:43:24,755 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 92c0b293dd887d05ae6f705c47b9c951/C in TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:24,755 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/86360001169a45a78d38183010f9cc23, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/079c6231155d4a169d394c30f629feac, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/4ea5d3cd381f4031a25bdd6452226e1b] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp, totalSize=36.3 K 2024-12-09T10:43:24,756 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 86360001169a45a78d38183010f9cc23, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1733740999203 2024-12-09T10:43:24,756 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 079c6231155d4a169d394c30f629feac, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1733741000403 2024-12-09T10:43:24,756 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ea5d3cd381f4031a25bdd6452226e1b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1733741002308 2024-12-09T10:43:24,763 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 92c0b293dd887d05ae6f705c47b9c951#C#compaction#484 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:43:24,763 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/C/b03be6c58be44294b35370243d809fdf is 50, key is test_row_0/C:col10/1733741003460/Put/seqid=0 2024-12-09T10:43:24,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742400_1576 (size=12847) 2024-12-09T10:43:24,807 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-09T10:43:24,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:24,808 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing 92c0b293dd887d05ae6f705c47b9c951 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-09T10:43:24,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 92c0b293dd887d05ae6f705c47b9c951, store=A 2024-12-09T10:43:24,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:43:24,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 92c0b293dd887d05ae6f705c47b9c951, store=B 2024-12-09T10:43:24,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:43:24,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 92c0b293dd887d05ae6f705c47b9c951, store=C 2024-12-09T10:43:24,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:43:24,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412098cc62f0a7f5d49b6afacbb74908bbd74_92c0b293dd887d05ae6f705c47b9c951 is 50, key is test_row_0/A:col10/1733741003539/Put/seqid=0 2024-12-09T10:43:24,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742401_1577 (size=12454) 2024-12-09T10:43:24,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-09T10:43:25,150 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 92c0b293dd887d05ae6f705c47b9c951#A#compaction#483 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:43:25,151 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/A/1fef977241f84591b2a05b3283d74cd3 is 175, key is test_row_0/A:col10/1733741003460/Put/seqid=0 2024-12-09T10:43:25,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742402_1578 (size=31801) 2024-12-09T10:43:25,176 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/C/b03be6c58be44294b35370243d809fdf as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/b03be6c58be44294b35370243d809fdf 2024-12-09T10:43:25,181 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 92c0b293dd887d05ae6f705c47b9c951/C of 92c0b293dd887d05ae6f705c47b9c951 into b03be6c58be44294b35370243d809fdf(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:43:25,181 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 92c0b293dd887d05ae6f705c47b9c951: 2024-12-09T10:43:25,181 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., storeName=92c0b293dd887d05ae6f705c47b9c951/C, priority=13, startTime=1733741004723; duration=0sec 2024-12-09T10:43:25,181 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:43:25,181 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 92c0b293dd887d05ae6f705c47b9c951:C 2024-12-09T10:43:25,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,221 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412098cc62f0a7f5d49b6afacbb74908bbd74_92c0b293dd887d05ae6f705c47b9c951 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412098cc62f0a7f5d49b6afacbb74908bbd74_92c0b293dd887d05ae6f705c47b9c951 2024-12-09T10:43:25,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/A/fd07b35d9285420f88b680024694e526, store: [table=TestAcidGuarantees family=A region=92c0b293dd887d05ae6f705c47b9c951] 2024-12-09T10:43:25,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/A/fd07b35d9285420f88b680024694e526 is 175, key is test_row_0/A:col10/1733741003539/Put/seqid=0 2024-12-09T10:43:25,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742403_1579 (size=31255) 2024-12-09T10:43:25,226 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=309, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/A/fd07b35d9285420f88b680024694e526 2024-12-09T10:43:25,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/B/5f241400dc1544e793ef75a324c16484 is 50, key is test_row_0/B:col10/1733741003539/Put/seqid=0 2024-12-09T10:43:25,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742404_1580 (size=12301) 2024-12-09T10:43:25,559 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/A/1fef977241f84591b2a05b3283d74cd3 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/1fef977241f84591b2a05b3283d74cd3 2024-12-09T10:43:25,563 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 92c0b293dd887d05ae6f705c47b9c951/A of 92c0b293dd887d05ae6f705c47b9c951 into 1fef977241f84591b2a05b3283d74cd3(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:43:25,563 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 92c0b293dd887d05ae6f705c47b9c951: 2024-12-09T10:43:25,563 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., storeName=92c0b293dd887d05ae6f705c47b9c951/A, priority=13, startTime=1733741004722; duration=0sec 2024-12-09T10:43:25,563 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:43:25,563 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 92c0b293dd887d05ae6f705c47b9c951:A 2024-12-09T10:43:25,636 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/B/5f241400dc1544e793ef75a324c16484 2024-12-09T10:43:25,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/C/6026a918fa3043a8ae8419b1e7681138 is 50, key is test_row_0/C:col10/1733741003539/Put/seqid=0 2024-12-09T10:43:25,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742405_1581 (size=12301) 2024-12-09T10:43:25,646 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/C/6026a918fa3043a8ae8419b1e7681138 2024-12-09T10:43:25,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/A/fd07b35d9285420f88b680024694e526 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/fd07b35d9285420f88b680024694e526 2024-12-09T10:43:25,653 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/fd07b35d9285420f88b680024694e526, entries=150, sequenceid=309, filesize=30.5 K 2024-12-09T10:43:25,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/B/5f241400dc1544e793ef75a324c16484 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/5f241400dc1544e793ef75a324c16484 2024-12-09T10:43:25,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,656 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/5f241400dc1544e793ef75a324c16484, entries=150, sequenceid=309, filesize=12.0 K 2024-12-09T10:43:25,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/C/6026a918fa3043a8ae8419b1e7681138 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/6026a918fa3043a8ae8419b1e7681138 2024-12-09T10:43:25,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,660 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/6026a918fa3043a8ae8419b1e7681138, entries=150, sequenceid=309, filesize=12.0 K 2024-12-09T10:43:25,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,661 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=0 B/0 for 92c0b293dd887d05ae6f705c47b9c951 in 853ms, sequenceid=309, compaction requested=false 2024-12-09T10:43:25,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for 92c0b293dd887d05ae6f705c47b9c951: 2024-12-09T10:43:25,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:25,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-12-09T10:43:25,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-12-09T10:43:25,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,664 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-12-09T10:43:25,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,664 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9290 sec 2024-12-09T10:43:25,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,666 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 1.9350 sec 2024-12-09T10:43:25,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(8855): Flush requested on 92c0b293dd887d05ae6f705c47b9c951 2024-12-09T10:43:25,760 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 92c0b293dd887d05ae6f705c47b9c951 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-09T10:43:25,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 92c0b293dd887d05ae6f705c47b9c951, store=A 2024-12-09T10:43:25,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:43:25,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 92c0b293dd887d05ae6f705c47b9c951, store=B 2024-12-09T10:43:25,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:43:25,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 92c0b293dd887d05ae6f705c47b9c951, store=C 2024-12-09T10:43:25,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:43:25,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:25,769 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412090ee1f404e3e84508bec2d6c36ffc9286_92c0b293dd887d05ae6f705c47b9c951 is 50, key is test_row_0/A:col10/1733741005759/Put/seqid=0 2024-12-09T10:43:25,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742406_1582 (size=22618) 2024-12-09T10:43:25,858 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733741005792, max timestamp=1733741005854Num uniques 859; 2024-12-09T10:43:25,858 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T10:43:25,858 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733741005792, max timestamp=1733741005854Num uniques 859; 2024-12-09T10:43:25,858 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T10:43:25,858 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733741005792, max timestamp=1733741005854Num uniques 859; 2024-12-09T10:43:25,858 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T10:43:25,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-09T10:43:25,879 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T10:43:25,880 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-09T10:43:25,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees 2024-12-09T10:43:25,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-09T10:43:25,882 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T10:43:25,882 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T10:43:25,883 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T10:43:25,934 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-09T10:43:25,961 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:25,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60502 deadline: 1733741065955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:25,963 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:25,963 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:25,963 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:25,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-09T10:43:26,035 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-09T10:43:26,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:26,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. as already flushing 2024-12-09T10:43:26,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:26,035 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:43:26,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:43:26,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:43:26,075 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:26,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60502 deadline: 1733741066070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:26,075 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:26,075 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:26,075 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:26,187 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-09T10:43:26,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:26,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. as already flushing 2024-12-09T10:43:26,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:26,188 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:43:26,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:43:26,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:43:26,191 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:26,195 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412090ee1f404e3e84508bec2d6c36ffc9286_92c0b293dd887d05ae6f705c47b9c951 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412090ee1f404e3e84508bec2d6c36ffc9286_92c0b293dd887d05ae6f705c47b9c951 2024-12-09T10:43:26,196 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/A/7316ceecceb74e79a78a87446bd7418d, store: [table=TestAcidGuarantees family=A region=92c0b293dd887d05ae6f705c47b9c951] 2024-12-09T10:43:26,196 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/A/7316ceecceb74e79a78a87446bd7418d is 175, key is test_row_0/A:col10/1733741005759/Put/seqid=0 2024-12-09T10:43:26,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-09T10:43:26,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742407_1583 (size=66023) 2024-12-09T10:43:26,203 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=321, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/A/7316ceecceb74e79a78a87446bd7418d 2024-12-09T10:43:26,209 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/B/f89cb34afa254ad7a2e33d0e8811986b is 50, key is test_row_0/B:col10/1733741005759/Put/seqid=0 2024-12-09T10:43:26,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742408_1584 (size=12301) 2024-12-09T10:43:26,284 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:26,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60502 deadline: 1733741066280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:26,284 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:26,284 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:26,284 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:26,341 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-09T10:43:26,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:26,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. as already flushing 2024-12-09T10:43:26,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:26,341 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:43:26,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:43:26,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:43:26,494 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-09T10:43:26,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:26,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. as already flushing 2024-12-09T10:43:26,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:26,494 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:43:26,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:43:26,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:43:26,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-09T10:43:26,593 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:26,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60502 deadline: 1733741066590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:26,594 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:26,594 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:26,594 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:26,613 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/B/f89cb34afa254ad7a2e33d0e8811986b 2024-12-09T10:43:26,619 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/C/9c2140fb4625483ba65292e7345ddd29 is 50, key is test_row_0/C:col10/1733741005759/Put/seqid=0 2024-12-09T10:43:26,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742409_1585 (size=12301) 2024-12-09T10:43:26,647 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-09T10:43:26,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:26,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. as already flushing 2024-12-09T10:43:26,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:26,647 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:43:26,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:43:26,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:43:26,800 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-09T10:43:26,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:26,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. as already flushing 2024-12-09T10:43:26,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:26,801 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:43:26,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:43:26,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:43:26,953 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-09T10:43:26,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:26,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. as already flushing 2024-12-09T10:43:26,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:26,954 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:43:26,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:43:26,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T10:43:27,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-09T10:43:27,023 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/C/9c2140fb4625483ba65292e7345ddd29 2024-12-09T10:43:27,027 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/A/7316ceecceb74e79a78a87446bd7418d as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/7316ceecceb74e79a78a87446bd7418d 2024-12-09T10:43:27,030 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/7316ceecceb74e79a78a87446bd7418d, entries=350, sequenceid=321, filesize=64.5 K 2024-12-09T10:43:27,030 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/B/f89cb34afa254ad7a2e33d0e8811986b as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/f89cb34afa254ad7a2e33d0e8811986b 2024-12-09T10:43:27,033 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/f89cb34afa254ad7a2e33d0e8811986b, entries=150, sequenceid=321, filesize=12.0 K 2024-12-09T10:43:27,034 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/C/9c2140fb4625483ba65292e7345ddd29 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/9c2140fb4625483ba65292e7345ddd29 2024-12-09T10:43:27,036 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/9c2140fb4625483ba65292e7345ddd29, entries=150, sequenceid=321, filesize=12.0 K 2024-12-09T10:43:27,037 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 92c0b293dd887d05ae6f705c47b9c951 in 1277ms, sequenceid=321, compaction requested=true 2024-12-09T10:43:27,037 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 92c0b293dd887d05ae6f705c47b9c951: 2024-12-09T10:43:27,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 92c0b293dd887d05ae6f705c47b9c951:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T10:43:27,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:43:27,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 92c0b293dd887d05ae6f705c47b9c951:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T10:43:27,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:43:27,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 92c0b293dd887d05ae6f705c47b9c951:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T10:43:27,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T10:43:27,037 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T10:43:27,037 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T10:43:27,038 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T10:43:27,038 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 129079 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T10:43:27,038 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1541): 92c0b293dd887d05ae6f705c47b9c951/B is initiating minor compaction (all files) 2024-12-09T10:43:27,038 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1541): 92c0b293dd887d05ae6f705c47b9c951/A is initiating minor compaction (all files) 2024-12-09T10:43:27,038 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 92c0b293dd887d05ae6f705c47b9c951/A in TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:27,038 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 92c0b293dd887d05ae6f705c47b9c951/B in TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:27,038 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/1fef977241f84591b2a05b3283d74cd3, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/fd07b35d9285420f88b680024694e526, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/7316ceecceb74e79a78a87446bd7418d] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp, totalSize=126.1 K 2024-12-09T10:43:27,038 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/a84f584c5da84ce4b9a391cf1945842e, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/5f241400dc1544e793ef75a324c16484, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/f89cb34afa254ad7a2e33d0e8811986b] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp, totalSize=36.6 K 2024-12-09T10:43:27,038 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:27,038 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. files: [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/1fef977241f84591b2a05b3283d74cd3, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/fd07b35d9285420f88b680024694e526, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/7316ceecceb74e79a78a87446bd7418d] 2024-12-09T10:43:27,039 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting a84f584c5da84ce4b9a391cf1945842e, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1733741002308 2024-12-09T10:43:27,039 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1fef977241f84591b2a05b3283d74cd3, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1733741002308 2024-12-09T10:43:27,039 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f241400dc1544e793ef75a324c16484, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1733741003507 2024-12-09T10:43:27,039 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting fd07b35d9285420f88b680024694e526, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1733741003507 2024-12-09T10:43:27,039 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting f89cb34afa254ad7a2e33d0e8811986b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1733741005730 2024-12-09T10:43:27,039 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7316ceecceb74e79a78a87446bd7418d, keycount=350, bloomtype=ROW, size=64.5 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1733741005707 2024-12-09T10:43:27,046 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=92c0b293dd887d05ae6f705c47b9c951] 2024-12-09T10:43:27,048 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 92c0b293dd887d05ae6f705c47b9c951#B#compaction#491 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:43:27,048 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/B/1dc8f315c46d488b96af735aa24366bb is 50, key is test_row_0/B:col10/1733741005759/Put/seqid=0 2024-12-09T10:43:27,055 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120957fed24cfb334ad7b987e315b5a11035_92c0b293dd887d05ae6f705c47b9c951 store=[table=TestAcidGuarantees family=A region=92c0b293dd887d05ae6f705c47b9c951] 2024-12-09T10:43:27,057 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120957fed24cfb334ad7b987e315b5a11035_92c0b293dd887d05ae6f705c47b9c951, store=[table=TestAcidGuarantees family=A region=92c0b293dd887d05ae6f705c47b9c951] 2024-12-09T10:43:27,057 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120957fed24cfb334ad7b987e315b5a11035_92c0b293dd887d05ae6f705c47b9c951 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=92c0b293dd887d05ae6f705c47b9c951] 2024-12-09T10:43:27,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742410_1586 (size=12949) 2024-12-09T10:43:27,063 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/B/1dc8f315c46d488b96af735aa24366bb as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/1dc8f315c46d488b96af735aa24366bb 2024-12-09T10:43:27,067 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 92c0b293dd887d05ae6f705c47b9c951/B of 92c0b293dd887d05ae6f705c47b9c951 into 1dc8f315c46d488b96af735aa24366bb(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:43:27,067 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 92c0b293dd887d05ae6f705c47b9c951: 2024-12-09T10:43:27,067 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., storeName=92c0b293dd887d05ae6f705c47b9c951/B, priority=13, startTime=1733741007037; duration=0sec 2024-12-09T10:43:27,067 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T10:43:27,067 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 92c0b293dd887d05ae6f705c47b9c951:B 2024-12-09T10:43:27,067 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T10:43:27,069 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T10:43:27,069 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1541): 92c0b293dd887d05ae6f705c47b9c951/C is initiating minor compaction (all files) 2024-12-09T10:43:27,069 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 92c0b293dd887d05ae6f705c47b9c951/C in TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:27,069 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/b03be6c58be44294b35370243d809fdf, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/6026a918fa3043a8ae8419b1e7681138, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/9c2140fb4625483ba65292e7345ddd29] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp, totalSize=36.6 K 2024-12-09T10:43:27,069 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting b03be6c58be44294b35370243d809fdf, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1733741002308 2024-12-09T10:43:27,069 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 6026a918fa3043a8ae8419b1e7681138, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1733741003507 2024-12-09T10:43:27,070 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c2140fb4625483ba65292e7345ddd29, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1733741005730 2024-12-09T10:43:27,077 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 92c0b293dd887d05ae6f705c47b9c951#C#compaction#493 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:43:27,077 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/C/92a9e2264aa44af7bdac85a71034a695 is 50, key is test_row_0/C:col10/1733741005759/Put/seqid=0 2024-12-09T10:43:27,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742411_1587 (size=4469) 2024-12-09T10:43:27,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742412_1588 (size=12949) 2024-12-09T10:43:27,092 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/C/92a9e2264aa44af7bdac85a71034a695 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/92a9e2264aa44af7bdac85a71034a695 2024-12-09T10:43:27,096 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 92c0b293dd887d05ae6f705c47b9c951/C of 92c0b293dd887d05ae6f705c47b9c951 into 92a9e2264aa44af7bdac85a71034a695(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:43:27,096 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 92c0b293dd887d05ae6f705c47b9c951: 2024-12-09T10:43:27,096 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., storeName=92c0b293dd887d05ae6f705c47b9c951/C, priority=13, startTime=1733741007037; duration=0sec 2024-12-09T10:43:27,096 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:43:27,096 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 92c0b293dd887d05ae6f705c47b9c951:C 2024-12-09T10:43:27,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(8855): Flush requested on 92c0b293dd887d05ae6f705c47b9c951 2024-12-09T10:43:27,106 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-09T10:43:27,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:27,107 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2902): Flushing 92c0b293dd887d05ae6f705c47b9c951 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-12-09T10:43:27,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 92c0b293dd887d05ae6f705c47b9c951, store=A 2024-12-09T10:43:27,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-09T10:43:27,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 92c0b293dd887d05ae6f705c47b9c951, store=B 2024-12-09T10:43:27,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-09T10:43:27,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 92c0b293dd887d05ae6f705c47b9c951, store=C 2024-12-09T10:43:27,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-09T10:43:27,108 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. as already flushing 2024-12-09T10:43:27,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209cc4070a6f88747e980776e6c8ddb5eff_92c0b293dd887d05ae6f705c47b9c951 is 50, key is test_row_0/A:col10/1733741007103/Put/seqid=0 2024-12-09T10:43:27,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742413_1589 (size=12454) 2024-12-09T10:43:27,150 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:27,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60502 deadline: 1733741067145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:27,151 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:27,151 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:27,151 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:27,263 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:27,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60502 deadline: 1733741067260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:27,264 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:27,264 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:27,264 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:27,473 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:27,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60502 deadline: 1733741067470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:27,473 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:27,473 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:27,473 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:27,486 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 92c0b293dd887d05ae6f705c47b9c951#A#compaction#492 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:43:27,486 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/A/453aef8271d24df3b63deb7a9f83af89 is 175, key is test_row_0/A:col10/1733741005759/Put/seqid=0 2024-12-09T10:43:27,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742414_1590 (size=31903) 2024-12-09T10:43:27,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:27,538 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209cc4070a6f88747e980776e6c8ddb5eff_92c0b293dd887d05ae6f705c47b9c951 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209cc4070a6f88747e980776e6c8ddb5eff_92c0b293dd887d05ae6f705c47b9c951 2024-12-09T10:43:27,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/A/06beafdd19f84503b779909f7e69026e, store: [table=TestAcidGuarantees family=A region=92c0b293dd887d05ae6f705c47b9c951] 2024-12-09T10:43:27,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/A/06beafdd19f84503b779909f7e69026e is 175, key is test_row_0/A:col10/1733741007103/Put/seqid=0 2024-12-09T10:43:27,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742415_1591 (size=31255) 2024-12-09T10:43:27,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:27,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60502 deadline: 1733741067780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:27,783 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:27,783 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:27,783 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:27,894 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/A/453aef8271d24df3b63deb7a9f83af89 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/453aef8271d24df3b63deb7a9f83af89 2024-12-09T10:43:27,898 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 92c0b293dd887d05ae6f705c47b9c951/A of 92c0b293dd887d05ae6f705c47b9c951 into 453aef8271d24df3b63deb7a9f83af89(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:43:27,898 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 92c0b293dd887d05ae6f705c47b9c951: 2024-12-09T10:43:27,898 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., storeName=92c0b293dd887d05ae6f705c47b9c951/A, priority=13, startTime=1733741007037; duration=0sec 2024-12-09T10:43:27,899 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:43:27,899 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 92c0b293dd887d05ae6f705c47b9c951:A 2024-12-09T10:43:27,943 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=361, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/A/06beafdd19f84503b779909f7e69026e 2024-12-09T10:43:27,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/B/d780e2551af74785bc99d72f3d3ad316 is 50, key is test_row_0/B:col10/1733741007103/Put/seqid=0 2024-12-09T10:43:27,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742416_1592 (size=12301) 2024-12-09T10:43:28,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-09T10:43:28,292 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:28,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60502 deadline: 1733741068289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:28,293 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:28,293 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:28,293 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:28,354 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=361 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/B/d780e2551af74785bc99d72f3d3ad316 2024-12-09T10:43:28,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/C/0f57b19b90944c938b5b9ccc9c585649 is 50, key is test_row_0/C:col10/1733741007103/Put/seqid=0 2024-12-09T10:43:28,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742417_1593 (size=12301) 2024-12-09T10:43:28,372 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=361 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/C/0f57b19b90944c938b5b9ccc9c585649 2024-12-09T10:43:28,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/A/06beafdd19f84503b779909f7e69026e as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/06beafdd19f84503b779909f7e69026e 2024-12-09T10:43:28,382 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/06beafdd19f84503b779909f7e69026e, entries=150, sequenceid=361, filesize=30.5 K 2024-12-09T10:43:28,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/B/d780e2551af74785bc99d72f3d3ad316 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/d780e2551af74785bc99d72f3d3ad316 2024-12-09T10:43:28,386 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/d780e2551af74785bc99d72f3d3ad316, entries=150, sequenceid=361, filesize=12.0 K 2024-12-09T10:43:28,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/C/0f57b19b90944c938b5b9ccc9c585649 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/0f57b19b90944c938b5b9ccc9c585649 2024-12-09T10:43:28,390 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/0f57b19b90944c938b5b9ccc9c585649, entries=150, sequenceid=361, filesize=12.0 K 2024-12-09T10:43:28,391 INFO [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~405.56 KB/415296, currentSize=40.25 KB/41220 for 92c0b293dd887d05ae6f705c47b9c951 in 1285ms, sequenceid=361, compaction requested=false 2024-12-09T10:43:28,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2603): Flush status journal for 92c0b293dd887d05ae6f705c47b9c951: 2024-12-09T10:43:28,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:28,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ea43b39fc24:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=163 2024-12-09T10:43:28,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster(4169): Remote procedure done, pid=163 2024-12-09T10:43:28,395 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-12-09T10:43:28,395 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5100 sec 2024-12-09T10:43:28,397 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees in 2.5160 sec 2024-12-09T10:43:28,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(8855): Flush requested on 92c0b293dd887d05ae6f705c47b9c951 2024-12-09T10:43:28,581 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 92c0b293dd887d05ae6f705c47b9c951 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-09T10:43:28,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 92c0b293dd887d05ae6f705c47b9c951, store=A 2024-12-09T10:43:28,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:43:28,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 92c0b293dd887d05ae6f705c47b9c951, store=B 2024-12-09T10:43:28,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:43:28,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 92c0b293dd887d05ae6f705c47b9c951, store=C 2024-12-09T10:43:28,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:43:28,589 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209d6326909be424b6e8c89eb2030e5ed63_92c0b293dd887d05ae6f705c47b9c951 is 50, key is test_row_0/A:col10/1733741007132/Put/seqid=0 2024-12-09T10:43:28,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742418_1594 (size=12454) 2024-12-09T10:43:28,597 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:28,601 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209d6326909be424b6e8c89eb2030e5ed63_92c0b293dd887d05ae6f705c47b9c951 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209d6326909be424b6e8c89eb2030e5ed63_92c0b293dd887d05ae6f705c47b9c951 2024-12-09T10:43:28,602 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/A/f3f06660dda14254b63ea9e3b0c4288f, store: [table=TestAcidGuarantees family=A region=92c0b293dd887d05ae6f705c47b9c951] 2024-12-09T10:43:28,602 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/A/f3f06660dda14254b63ea9e3b0c4288f is 175, key is test_row_0/A:col10/1733741007132/Put/seqid=0 2024-12-09T10:43:28,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742419_1595 (size=31255) 2024-12-09T10:43:28,638 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733741008626, max timestamp=1733741008635Num uniques 859; 2024-12-09T10:43:28,638 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733741008626, max timestamp=1733741008635Num uniques 859; 2024-12-09T10:43:28,638 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T10:43:28,638 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T10:43:28,642 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733741008626, max timestamp=1733741008635Num uniques 859; 2024-12-09T10:43:28,642 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ea43b39fc24:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T10:43:28,670 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:28,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60518 deadline: 1733741068662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:28,670 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:28,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60526 deadline: 1733741068663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:28,670 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:28,670 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:28,670 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:28,671 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:28,671 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:28,671 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:28,676 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:28,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60494 deadline: 1733741068669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:28,676 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:28,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60464 deadline: 1733741068670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:28,676 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:28,676 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:28,676 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:28,677 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:28,677 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:28,677 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:28,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:28,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60526 deadline: 1733741068779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:28,784 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:28,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60518 deadline: 1733741068779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:28,784 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:28,784 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:28,784 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:28,784 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:28,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60494 deadline: 1733741068780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:28,784 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:28,784 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:28,784 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:28,784 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:28,784 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:28,784 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:28,784 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:28,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60464 deadline: 1733741068780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:28,784 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:28,785 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:28,785 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:28,921 INFO [Thread-2302 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2302. 2024-12-09T10:43:28,921 DEBUG [Thread-2302 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T10:43:28,921 DEBUG [Thread-2302 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:43:28,921 DEBUG [Thread-2302 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:43:28,921 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T10:43:28,923 INFO [Thread-2298 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2298. 2024-12-09T10:43:28,923 DEBUG [Thread-2298 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T10:43:28,923 DEBUG [Thread-2298 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:43:28,923 DEBUG [Thread-2298 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:43:28,923 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T10:43:28,923 INFO [Thread-2300 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2300. 2024-12-09T10:43:28,923 DEBUG [Thread-2300 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T10:43:28,923 DEBUG [Thread-2300 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:43:28,923 DEBUG [Thread-2300 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:43:28,923 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T10:43:28,926 INFO [Thread-2306 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2306. 2024-12-09T10:43:28,926 DEBUG [Thread-2306 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T10:43:28,926 DEBUG [Thread-2306 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:43:28,926 DEBUG [Thread-2306 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:43:28,926 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T10:43:28,927 INFO [Thread-2304 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2304. 2024-12-09T10:43:28,927 DEBUG [Thread-2304 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T10:43:28,927 DEBUG [Thread-2304 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:43:28,927 DEBUG [Thread-2304 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:43:28,927 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T10:43:28,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:28,990 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:28,990 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:28,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60526 deadline: 1733741068990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:28,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60518 deadline: 1733741068990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:28,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60494 deadline: 1733741068990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:28,990 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:28,990 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:28,990 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:28,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60464 deadline: 1733741068990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:28,990 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:28,990 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:28,990 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:28,990 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:28,990 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:28,990 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:28,990 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:28,991 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:28,991 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:28,991 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:29,011 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=372, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/A/f3f06660dda14254b63ea9e3b0c4288f 2024-12-09T10:43:29,016 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/B/34ad5b83260d47cc8c41094b52f5da65 is 50, key is test_row_0/B:col10/1733741007132/Put/seqid=0 2024-12-09T10:43:29,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742420_1596 (size=12301) 2024-12-09T10:43:29,300 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:29,300 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:29,300 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:29,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60518 deadline: 1733741069300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:29,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60494 deadline: 1733741069299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:29,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60502 deadline: 1733741069299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:29,300 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:29,300 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:29,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60464 deadline: 1733741069300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:29,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60526 deadline: 1733741069300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:29,300 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:29,300 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:29,300 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:29,300 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:29,300 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:29,300 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:29,300 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:29,300 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:29,300 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:29,300 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:29,301 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:29,301 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:29,300 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 7ea43b39fc24,36101,1733740827678 for 'test_row_2' in 92c0b293dd887d05ae6f705c47b9c951 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2155 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T10:43:29,301 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:29,301 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:29,301 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:29,420 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/B/34ad5b83260d47cc8c41094b52f5da65 2024-12-09T10:43:29,426 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/C/fecc056859814a73ad2e61a1a69c04ed is 50, key is test_row_0/C:col10/1733741007132/Put/seqid=0 2024-12-09T10:43:29,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742421_1597 (size=12301) 2024-12-09T10:43:29,810 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:29,810 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:29,810 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:29,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60518 deadline: 1733741069810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:29,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60526 deadline: 1733741069810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:29,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60494 deadline: 1733741069810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:29,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T10:43:29,811 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:29,811 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:29,811 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:29,811 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:29,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36101 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60464 deadline: 1733741069810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:29,811 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:29,811 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:29,811 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:29,811 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:29,811 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:29,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 , the old value is region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:29,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T10:43:29,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., hostname=7ea43b39fc24,36101,1733740827678, seqNum=5 because the exception is null or not the one we care about 2024-12-09T10:43:29,829 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/C/fecc056859814a73ad2e61a1a69c04ed 2024-12-09T10:43:29,833 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/A/f3f06660dda14254b63ea9e3b0c4288f as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/f3f06660dda14254b63ea9e3b0c4288f 2024-12-09T10:43:29,839 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/f3f06660dda14254b63ea9e3b0c4288f, entries=150, sequenceid=372, filesize=30.5 K 2024-12-09T10:43:29,839 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/B/34ad5b83260d47cc8c41094b52f5da65 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/34ad5b83260d47cc8c41094b52f5da65 2024-12-09T10:43:29,842 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/34ad5b83260d47cc8c41094b52f5da65, entries=150, sequenceid=372, filesize=12.0 K 2024-12-09T10:43:29,843 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/C/fecc056859814a73ad2e61a1a69c04ed as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/fecc056859814a73ad2e61a1a69c04ed 2024-12-09T10:43:29,846 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/fecc056859814a73ad2e61a1a69c04ed, entries=150, sequenceid=372, filesize=12.0 K 2024-12-09T10:43:29,847 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for 92c0b293dd887d05ae6f705c47b9c951 in 1266ms, sequenceid=372, compaction requested=true 2024-12-09T10:43:29,847 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 92c0b293dd887d05ae6f705c47b9c951: 2024-12-09T10:43:29,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 92c0b293dd887d05ae6f705c47b9c951:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T10:43:29,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:43:29,847 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T10:43:29,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 92c0b293dd887d05ae6f705c47b9c951:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T10:43:29,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:43:29,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 92c0b293dd887d05ae6f705c47b9c951:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T10:43:29,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T10:43:29,847 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T10:43:29,848 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94413 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T10:43:29,848 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T10:43:29,848 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1541): 92c0b293dd887d05ae6f705c47b9c951/B is initiating minor compaction (all files) 2024-12-09T10:43:29,848 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1541): 92c0b293dd887d05ae6f705c47b9c951/A is initiating minor compaction (all files) 2024-12-09T10:43:29,848 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 92c0b293dd887d05ae6f705c47b9c951/B in TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:29,848 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 92c0b293dd887d05ae6f705c47b9c951/A in TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:29,848 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/1dc8f315c46d488b96af735aa24366bb, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/d780e2551af74785bc99d72f3d3ad316, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/34ad5b83260d47cc8c41094b52f5da65] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp, totalSize=36.7 K 2024-12-09T10:43:29,848 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/453aef8271d24df3b63deb7a9f83af89, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/06beafdd19f84503b779909f7e69026e, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/f3f06660dda14254b63ea9e3b0c4288f] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp, totalSize=92.2 K 2024-12-09T10:43:29,848 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:29,848 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. files: [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/453aef8271d24df3b63deb7a9f83af89, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/06beafdd19f84503b779909f7e69026e, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/f3f06660dda14254b63ea9e3b0c4288f] 2024-12-09T10:43:29,848 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 1dc8f315c46d488b96af735aa24366bb, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1733741005730 2024-12-09T10:43:29,849 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 453aef8271d24df3b63deb7a9f83af89, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1733741005730 2024-12-09T10:43:29,850 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting d780e2551af74785bc99d72f3d3ad316, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1733741005938 2024-12-09T10:43:29,850 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 06beafdd19f84503b779909f7e69026e, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1733741005938 2024-12-09T10:43:29,850 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 34ad5b83260d47cc8c41094b52f5da65, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1733741007113 2024-12-09T10:43:29,850 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] compactions.Compactor(225): Compacting f3f06660dda14254b63ea9e3b0c4288f, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1733741007113 2024-12-09T10:43:29,856 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 92c0b293dd887d05ae6f705c47b9c951#B#compaction#500 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:43:29,856 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/B/75bc29876a944a188a3b159a6518ac21 is 50, key is test_row_0/B:col10/1733741007132/Put/seqid=0 2024-12-09T10:43:29,857 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=92c0b293dd887d05ae6f705c47b9c951] 2024-12-09T10:43:29,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742422_1598 (size=13051) 2024-12-09T10:43:29,862 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241209adeb38abf2684e148b205ceb7e99eaa2_92c0b293dd887d05ae6f705c47b9c951 store=[table=TestAcidGuarantees family=A region=92c0b293dd887d05ae6f705c47b9c951] 2024-12-09T10:43:29,867 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241209adeb38abf2684e148b205ceb7e99eaa2_92c0b293dd887d05ae6f705c47b9c951, store=[table=TestAcidGuarantees family=A region=92c0b293dd887d05ae6f705c47b9c951] 2024-12-09T10:43:29,867 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209adeb38abf2684e148b205ceb7e99eaa2_92c0b293dd887d05ae6f705c47b9c951 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=92c0b293dd887d05ae6f705c47b9c951] 2024-12-09T10:43:29,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742423_1599 (size=4469) 2024-12-09T10:43:30,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-09T10:43:30,050 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T10:43:30,263 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/B/75bc29876a944a188a3b159a6518ac21 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/75bc29876a944a188a3b159a6518ac21 2024-12-09T10:43:30,267 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 92c0b293dd887d05ae6f705c47b9c951/B of 92c0b293dd887d05ae6f705c47b9c951 into 75bc29876a944a188a3b159a6518ac21(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:43:30,267 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 92c0b293dd887d05ae6f705c47b9c951: 2024-12-09T10:43:30,267 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., storeName=92c0b293dd887d05ae6f705c47b9c951/B, priority=13, startTime=1733741009847; duration=0sec 2024-12-09T10:43:30,267 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T10:43:30,267 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 92c0b293dd887d05ae6f705c47b9c951:B 2024-12-09T10:43:30,267 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T10:43:30,268 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T10:43:30,268 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1541): 92c0b293dd887d05ae6f705c47b9c951/C is initiating minor compaction (all files) 2024-12-09T10:43:30,268 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 92c0b293dd887d05ae6f705c47b9c951/C in TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:30,268 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/92a9e2264aa44af7bdac85a71034a695, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/0f57b19b90944c938b5b9ccc9c585649, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/fecc056859814a73ad2e61a1a69c04ed] into tmpdir=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp, totalSize=36.7 K 2024-12-09T10:43:30,268 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 92a9e2264aa44af7bdac85a71034a695, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1733741005730 2024-12-09T10:43:30,268 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f57b19b90944c938b5b9ccc9c585649, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1733741005938 2024-12-09T10:43:30,268 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] compactions.Compactor(225): Compacting fecc056859814a73ad2e61a1a69c04ed, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1733741007113 2024-12-09T10:43:30,272 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 92c0b293dd887d05ae6f705c47b9c951#A#compaction#501 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:43:30,272 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/A/b0272bc4d07e4c6aa8d46b7b9cf377ea is 175, key is test_row_0/A:col10/1733741007132/Put/seqid=0 2024-12-09T10:43:30,273 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 92c0b293dd887d05ae6f705c47b9c951#C#compaction#502 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T10:43:30,274 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/C/b524cccac9584583944c2182ac33bb12 is 50, key is test_row_0/C:col10/1733741007132/Put/seqid=0 2024-12-09T10:43:30,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742424_1600 (size=32005) 2024-12-09T10:43:30,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742425_1601 (size=13051) 2024-12-09T10:43:30,283 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/C/b524cccac9584583944c2182ac33bb12 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/b524cccac9584583944c2182ac33bb12 2024-12-09T10:43:30,286 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 92c0b293dd887d05ae6f705c47b9c951/C of 92c0b293dd887d05ae6f705c47b9c951 into b524cccac9584583944c2182ac33bb12(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:43:30,286 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 92c0b293dd887d05ae6f705c47b9c951: 2024-12-09T10:43:30,286 INFO [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., storeName=92c0b293dd887d05ae6f705c47b9c951/C, priority=13, startTime=1733741009847; duration=0sec 2024-12-09T10:43:30,286 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:43:30,286 DEBUG [RS:0;7ea43b39fc24:36101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 92c0b293dd887d05ae6f705c47b9c951:C 2024-12-09T10:43:30,683 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/A/b0272bc4d07e4c6aa8d46b7b9cf377ea as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/b0272bc4d07e4c6aa8d46b7b9cf377ea 2024-12-09T10:43:30,686 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 92c0b293dd887d05ae6f705c47b9c951/A of 92c0b293dd887d05ae6f705c47b9c951 into b0272bc4d07e4c6aa8d46b7b9cf377ea(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T10:43:30,686 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 92c0b293dd887d05ae6f705c47b9c951: 2024-12-09T10:43:30,686 INFO [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., storeName=92c0b293dd887d05ae6f705c47b9c951/A, priority=13, startTime=1733741009847; duration=0sec 2024-12-09T10:43:30,686 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T10:43:30,686 DEBUG [RS:0;7ea43b39fc24:36101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 92c0b293dd887d05ae6f705c47b9c951:A 2024-12-09T10:43:30,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36101 {}] regionserver.HRegion(8855): Flush requested on 92c0b293dd887d05ae6f705c47b9c951 2024-12-09T10:43:30,821 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 92c0b293dd887d05ae6f705c47b9c951 3/3 column families, dataSize=154.31 KB heapSize=440.77 KB 2024-12-09T10:43:30,822 INFO [Thread-2295 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2295. 2024-12-09T10:43:30,822 INFO [Thread-2289 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2289. 2024-12-09T10:43:30,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 92c0b293dd887d05ae6f705c47b9c951, store=A 2024-12-09T10:43:30,822 DEBUG [Thread-2295 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T10:43:30,822 DEBUG [Thread-2289 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T10:43:30,822 DEBUG [Thread-2295 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:43:30,822 DEBUG [Thread-2289 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:43:30,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-09T10:43:30,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 92c0b293dd887d05ae6f705c47b9c951, store=B 2024-12-09T10:43:30,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-09T10:43:30,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 92c0b293dd887d05ae6f705c47b9c951, store=C 2024-12-09T10:43:30,822 DEBUG [Thread-2289 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:43:30,822 DEBUG [Thread-2295 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:43:30,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-09T10:43:30,822 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T10:43:30,822 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T10:43:30,827 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120944cbf5dc179c4a518be0872fe9993406_92c0b293dd887d05ae6f705c47b9c951 is 50, key is test_row_0/A:col10/1733741008669/Put/seqid=0 2024-12-09T10:43:30,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742426_1602 (size=12454) 2024-12-09T10:43:30,831 INFO [Thread-2293 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2293. 2024-12-09T10:43:30,831 INFO [Thread-2287 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2287. 2024-12-09T10:43:30,831 DEBUG [Thread-2293 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T10:43:30,831 DEBUG [Thread-2293 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:43:30,831 DEBUG [Thread-2287 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T10:43:30,831 DEBUG [Thread-2287 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:43:30,831 DEBUG [Thread-2293 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:43:30,831 DEBUG [Thread-2287 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:43:30,831 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T10:43:30,831 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T10:43:31,231 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:31,234 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120944cbf5dc179c4a518be0872fe9993406_92c0b293dd887d05ae6f705c47b9c951 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120944cbf5dc179c4a518be0872fe9993406_92c0b293dd887d05ae6f705c47b9c951 2024-12-09T10:43:31,234 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/A/31518a620bbe47a5b820c9579efddcb3, store: [table=TestAcidGuarantees family=A region=92c0b293dd887d05ae6f705c47b9c951] 2024-12-09T10:43:31,235 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/A/31518a620bbe47a5b820c9579efddcb3 is 175, key is test_row_0/A:col10/1733741008669/Put/seqid=0 2024-12-09T10:43:31,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742427_1603 (size=31255) 2024-12-09T10:43:31,321 INFO [Thread-2291 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2291. 2024-12-09T10:43:31,321 DEBUG [Thread-2291 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T10:43:31,321 DEBUG [Thread-2291 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:43:31,321 DEBUG [Thread-2291 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:43:31,321 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T10:43:31,321 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-09T10:43:31,321 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 62 2024-12-09T10:43:31,321 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 57 2024-12-09T10:43:31,321 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 103 2024-12-09T10:43:31,321 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 61 2024-12-09T10:43:31,321 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 58 2024-12-09T10:43:31,321 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-09T10:43:31,321 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2851 2024-12-09T10:43:31,321 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2809 2024-12-09T10:43:31,321 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2851 2024-12-09T10:43:31,321 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2830 2024-12-09T10:43:31,321 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2807 2024-12-09T10:43:31,322 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-09T10:43:31,322 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-09T10:43:31,322 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T10:43:31,322 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:43:31,322 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:43:31,322 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T10:43:31,322 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-12-09T10:43:31,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-09T10:43:31,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-09T10:43:31,325 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733741011325"}]},"ts":"1733741011325"} 2024-12-09T10:43:31,326 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-09T10:43:31,326 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-09T10:43:31,327 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-09T10:43:31,328 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=166, ppid=165, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=92c0b293dd887d05ae6f705c47b9c951, UNASSIGN}] 2024-12-09T10:43:31,328 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=166, ppid=165, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=92c0b293dd887d05ae6f705c47b9c951, UNASSIGN 2024-12-09T10:43:31,329 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=166 updating hbase:meta row=92c0b293dd887d05ae6f705c47b9c951, regionState=CLOSING, regionLocation=7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:31,330 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=166, ppid=165, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=92c0b293dd887d05ae6f705c47b9c951, UNASSIGN because future has completed 2024-12-09T10:43:31,330 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-09T10:43:31,330 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE, hasLock=false; CloseRegionProcedure 92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678}] 2024-12-09T10:43:31,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-09T10:43:31,482 INFO [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] handler.UnassignRegionHandler(122): Close 92c0b293dd887d05ae6f705c47b9c951 2024-12-09T10:43:31,482 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-09T10:43:31,482 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1722): Closing 92c0b293dd887d05ae6f705c47b9c951, disabling compactions & flushes 2024-12-09T10:43:31,482 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:31,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-09T10:43:31,639 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=414, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/A/31518a620bbe47a5b820c9579efddcb3 2024-12-09T10:43:31,645 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/B/fa706f5fcb584566993e2171df311645 is 50, key is test_row_0/B:col10/1733741008669/Put/seqid=0 2024-12-09T10:43:31,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742428_1604 (size=12301) 2024-12-09T10:43:31,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-09T10:43:32,048 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=414 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/B/fa706f5fcb584566993e2171df311645 2024-12-09T10:43:32,053 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/C/b7b91faae7ad48fcafee25b5cfee82a0 is 50, key is test_row_0/C:col10/1733741008669/Put/seqid=0 2024-12-09T10:43:32,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742429_1605 (size=12301) 2024-12-09T10:43:32,457 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=414 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/C/b7b91faae7ad48fcafee25b5cfee82a0 2024-12-09T10:43:32,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-09T10:43:32,461 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/A/31518a620bbe47a5b820c9579efddcb3 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/31518a620bbe47a5b820c9579efddcb3 2024-12-09T10:43:32,464 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/31518a620bbe47a5b820c9579efddcb3, entries=150, sequenceid=414, filesize=30.5 K 2024-12-09T10:43:32,464 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/B/fa706f5fcb584566993e2171df311645 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/fa706f5fcb584566993e2171df311645 2024-12-09T10:43:32,467 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/fa706f5fcb584566993e2171df311645, entries=150, sequenceid=414, filesize=12.0 K 2024-12-09T10:43:32,467 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/C/b7b91faae7ad48fcafee25b5cfee82a0 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/b7b91faae7ad48fcafee25b5cfee82a0 2024-12-09T10:43:32,470 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/b7b91faae7ad48fcafee25b5cfee82a0, entries=150, sequenceid=414, filesize=12.0 K 2024-12-09T10:43:32,471 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~440.72 KB/451296, currentSize=20.13 KB/20610 for 92c0b293dd887d05ae6f705c47b9c951 in 1650ms, sequenceid=414, compaction requested=false 2024-12-09T10:43:32,471 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 92c0b293dd887d05ae6f705c47b9c951: 2024-12-09T10:43:32,471 INFO [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:32,471 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:32,471 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. after waiting 0 ms 2024-12-09T10:43:32,471 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:32,471 INFO [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(2902): Flushing 92c0b293dd887d05ae6f705c47b9c951 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-12-09T10:43:32,471 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 92c0b293dd887d05ae6f705c47b9c951, store=A 2024-12-09T10:43:32,471 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:43:32,471 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 92c0b293dd887d05ae6f705c47b9c951, store=B 2024-12-09T10:43:32,471 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:43:32,471 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 92c0b293dd887d05ae6f705c47b9c951, store=C 2024-12-09T10:43:32,471 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T10:43:32,475 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120993ca667866f642a6b6e2441f08759c36_92c0b293dd887d05ae6f705c47b9c951 is 50, key is test_row_0/A:col10/1733741010830/Put/seqid=0 2024-12-09T10:43:32,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742430_1606 (size=9914) 2024-12-09T10:43:32,879 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T10:43:32,882 INFO [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120993ca667866f642a6b6e2441f08759c36_92c0b293dd887d05ae6f705c47b9c951 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120993ca667866f642a6b6e2441f08759c36_92c0b293dd887d05ae6f705c47b9c951 2024-12-09T10:43:32,882 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/A/633e2eaa88a341d881f1ae2268f00fee, store: [table=TestAcidGuarantees family=A region=92c0b293dd887d05ae6f705c47b9c951] 2024-12-09T10:43:32,883 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/A/633e2eaa88a341d881f1ae2268f00fee is 175, key is test_row_0/A:col10/1733741010830/Put/seqid=0 2024-12-09T10:43:32,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742431_1607 (size=22561) 2024-12-09T10:43:33,286 INFO [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=420, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/A/633e2eaa88a341d881f1ae2268f00fee 2024-12-09T10:43:33,292 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/B/33b60c18dea044fba1620a0cbb22a45b is 50, key is test_row_0/B:col10/1733741010830/Put/seqid=0 2024-12-09T10:43:33,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742432_1608 (size=9857) 2024-12-09T10:43:33,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-09T10:43:33,695 INFO [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=420 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/B/33b60c18dea044fba1620a0cbb22a45b 2024-12-09T10:43:33,700 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/C/3a26200acebe48b387989e00cf5cc259 is 50, key is test_row_0/C:col10/1733741010830/Put/seqid=0 2024-12-09T10:43:33,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742433_1609 (size=9857) 2024-12-09T10:43:34,104 INFO [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=420 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/C/3a26200acebe48b387989e00cf5cc259 2024-12-09T10:43:34,107 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/A/633e2eaa88a341d881f1ae2268f00fee as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/633e2eaa88a341d881f1ae2268f00fee 2024-12-09T10:43:34,110 INFO [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/633e2eaa88a341d881f1ae2268f00fee, entries=100, sequenceid=420, filesize=22.0 K 2024-12-09T10:43:34,110 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/B/33b60c18dea044fba1620a0cbb22a45b as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/33b60c18dea044fba1620a0cbb22a45b 2024-12-09T10:43:34,112 INFO [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/33b60c18dea044fba1620a0cbb22a45b, entries=100, sequenceid=420, filesize=9.6 K 2024-12-09T10:43:34,113 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/.tmp/C/3a26200acebe48b387989e00cf5cc259 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/3a26200acebe48b387989e00cf5cc259 2024-12-09T10:43:34,115 INFO [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/3a26200acebe48b387989e00cf5cc259, entries=100, sequenceid=420, filesize=9.6 K 2024-12-09T10:43:34,116 INFO [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 92c0b293dd887d05ae6f705c47b9c951 in 1645ms, sequenceid=420, compaction requested=true 2024-12-09T10:43:34,116 DEBUG [StoreCloser-TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/dc7da36494e34fb1921749692776ea6f, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/23b8ba829cf343628caa96e45fe9d3d2, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/d60dd66537d740f69f107cffc071d6aa, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/adc7d04859af409a8392ab390d0db49c, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/5f82811dc8b04a4e83fd1f45bf809ed1, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/7f5217c3ade4443d9117f24a7ec728be, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/1e1c6f08f05b4a87ba4205353ba6a9c8, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/7d4be0511f014f8e95f988850c2d3c15, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/ebc6b4168c9e416d85b123e7e41d1021, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/c2ecf4bd12cd4867b4fea00bb3d76cb5, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/ca2de9012aa54cc489fcf74c58edc76f, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/6d442efaf1784ee98612b5b5d7710035, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/f30ff041cda942458022fbb6ed798a8c, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/8778a840fb5f4e61ab802ecb2897f9f6, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/98c0729217644a6d86c1b14e412e12db, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/5b71d7f4ee1c4788ba845606e0e3c9d2, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/1fef977241f84591b2a05b3283d74cd3, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/fd07b35d9285420f88b680024694e526, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/7316ceecceb74e79a78a87446bd7418d, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/453aef8271d24df3b63deb7a9f83af89, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/06beafdd19f84503b779909f7e69026e, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/f3f06660dda14254b63ea9e3b0c4288f] to archive 2024-12-09T10:43:34,117 DEBUG [StoreCloser-TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T10:43:34,118 DEBUG [StoreCloser-TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/dc7da36494e34fb1921749692776ea6f to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/dc7da36494e34fb1921749692776ea6f 2024-12-09T10:43:34,119 DEBUG [StoreCloser-TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/23b8ba829cf343628caa96e45fe9d3d2 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/23b8ba829cf343628caa96e45fe9d3d2 2024-12-09T10:43:34,120 DEBUG [StoreCloser-TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/d60dd66537d740f69f107cffc071d6aa to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/d60dd66537d740f69f107cffc071d6aa 2024-12-09T10:43:34,121 DEBUG [StoreCloser-TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/adc7d04859af409a8392ab390d0db49c to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/adc7d04859af409a8392ab390d0db49c 2024-12-09T10:43:34,121 DEBUG [StoreCloser-TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/5f82811dc8b04a4e83fd1f45bf809ed1 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/5f82811dc8b04a4e83fd1f45bf809ed1 2024-12-09T10:43:34,122 DEBUG [StoreCloser-TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/7f5217c3ade4443d9117f24a7ec728be to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/7f5217c3ade4443d9117f24a7ec728be 2024-12-09T10:43:34,123 DEBUG [StoreCloser-TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/1e1c6f08f05b4a87ba4205353ba6a9c8 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/1e1c6f08f05b4a87ba4205353ba6a9c8 2024-12-09T10:43:34,124 DEBUG [StoreCloser-TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/7d4be0511f014f8e95f988850c2d3c15 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/7d4be0511f014f8e95f988850c2d3c15 2024-12-09T10:43:34,125 DEBUG [StoreCloser-TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/ebc6b4168c9e416d85b123e7e41d1021 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/ebc6b4168c9e416d85b123e7e41d1021 2024-12-09T10:43:34,125 DEBUG [StoreCloser-TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/c2ecf4bd12cd4867b4fea00bb3d76cb5 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/c2ecf4bd12cd4867b4fea00bb3d76cb5 2024-12-09T10:43:34,126 DEBUG [StoreCloser-TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/ca2de9012aa54cc489fcf74c58edc76f to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/ca2de9012aa54cc489fcf74c58edc76f 2024-12-09T10:43:34,127 DEBUG [StoreCloser-TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/6d442efaf1784ee98612b5b5d7710035 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/6d442efaf1784ee98612b5b5d7710035 2024-12-09T10:43:34,128 DEBUG [StoreCloser-TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/f30ff041cda942458022fbb6ed798a8c to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/f30ff041cda942458022fbb6ed798a8c 2024-12-09T10:43:34,129 DEBUG [StoreCloser-TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/8778a840fb5f4e61ab802ecb2897f9f6 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/8778a840fb5f4e61ab802ecb2897f9f6 2024-12-09T10:43:34,129 DEBUG [StoreCloser-TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/98c0729217644a6d86c1b14e412e12db to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/98c0729217644a6d86c1b14e412e12db 2024-12-09T10:43:34,130 DEBUG [StoreCloser-TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/5b71d7f4ee1c4788ba845606e0e3c9d2 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/5b71d7f4ee1c4788ba845606e0e3c9d2 2024-12-09T10:43:34,131 DEBUG [StoreCloser-TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/1fef977241f84591b2a05b3283d74cd3 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/1fef977241f84591b2a05b3283d74cd3 2024-12-09T10:43:34,132 DEBUG [StoreCloser-TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/fd07b35d9285420f88b680024694e526 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/fd07b35d9285420f88b680024694e526 2024-12-09T10:43:34,133 DEBUG [StoreCloser-TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/7316ceecceb74e79a78a87446bd7418d to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/7316ceecceb74e79a78a87446bd7418d 2024-12-09T10:43:34,134 DEBUG [StoreCloser-TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/453aef8271d24df3b63deb7a9f83af89 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/453aef8271d24df3b63deb7a9f83af89 2024-12-09T10:43:34,134 DEBUG [StoreCloser-TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/06beafdd19f84503b779909f7e69026e to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/06beafdd19f84503b779909f7e69026e 2024-12-09T10:43:34,135 DEBUG [StoreCloser-TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/f3f06660dda14254b63ea9e3b0c4288f to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/f3f06660dda14254b63ea9e3b0c4288f 2024-12-09T10:43:34,136 DEBUG [StoreCloser-TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/e252a9a06c3c47608ddaa35c36e798bc, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/ce55eca3e6e640cb9d810de214302669, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/0de7deafa4b047f5955b9d92716b7d8b, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/dc3d6d4bf80e4ee49d99ed2eff703360, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/3597e8a7c1774b9a8da80008b00ea2d2, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/3c04b67b06674529bc752e4cb546cd38, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/36d31d8e091f4f8b8f0c295903d9e83b, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/6192782f6c42497094580729231f4a4d, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/07072ac26e9b4995aba7c749e9c85570, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/97951b05faf74c9fbfbc59569daacaac, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/d8c3546531234dd5a54dafc4d7bb2053, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/d45462eac83e493f99272b1d5467e24b, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/b434797b8ca5484eaea5cda03c94e994, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/dc507977e37e4a629ad46e4272127702, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/b545338a8d4f4120aede397c7c1e031c, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/a84f584c5da84ce4b9a391cf1945842e, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/20843e37ceab4c24bbf55d56416a1859, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/5f241400dc1544e793ef75a324c16484, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/1dc8f315c46d488b96af735aa24366bb, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/f89cb34afa254ad7a2e33d0e8811986b, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/d780e2551af74785bc99d72f3d3ad316, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/34ad5b83260d47cc8c41094b52f5da65] to archive 2024-12-09T10:43:34,137 DEBUG [StoreCloser-TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T10:43:34,138 DEBUG [StoreCloser-TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/e252a9a06c3c47608ddaa35c36e798bc to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/e252a9a06c3c47608ddaa35c36e798bc 2024-12-09T10:43:34,139 DEBUG [StoreCloser-TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/ce55eca3e6e640cb9d810de214302669 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/ce55eca3e6e640cb9d810de214302669 2024-12-09T10:43:34,140 DEBUG [StoreCloser-TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/0de7deafa4b047f5955b9d92716b7d8b to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/0de7deafa4b047f5955b9d92716b7d8b 2024-12-09T10:43:34,140 DEBUG [StoreCloser-TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/dc3d6d4bf80e4ee49d99ed2eff703360 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/dc3d6d4bf80e4ee49d99ed2eff703360 2024-12-09T10:43:34,141 DEBUG [StoreCloser-TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/3597e8a7c1774b9a8da80008b00ea2d2 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/3597e8a7c1774b9a8da80008b00ea2d2 2024-12-09T10:43:34,142 DEBUG [StoreCloser-TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/3c04b67b06674529bc752e4cb546cd38 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/3c04b67b06674529bc752e4cb546cd38 2024-12-09T10:43:34,143 DEBUG [StoreCloser-TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/36d31d8e091f4f8b8f0c295903d9e83b to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/36d31d8e091f4f8b8f0c295903d9e83b 2024-12-09T10:43:34,144 DEBUG [StoreCloser-TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/6192782f6c42497094580729231f4a4d to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/6192782f6c42497094580729231f4a4d 2024-12-09T10:43:34,144 DEBUG [StoreCloser-TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/07072ac26e9b4995aba7c749e9c85570 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/07072ac26e9b4995aba7c749e9c85570 2024-12-09T10:43:34,145 DEBUG [StoreCloser-TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/97951b05faf74c9fbfbc59569daacaac to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/97951b05faf74c9fbfbc59569daacaac 2024-12-09T10:43:34,146 DEBUG [StoreCloser-TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/d8c3546531234dd5a54dafc4d7bb2053 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/d8c3546531234dd5a54dafc4d7bb2053 2024-12-09T10:43:34,147 DEBUG [StoreCloser-TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/d45462eac83e493f99272b1d5467e24b to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/d45462eac83e493f99272b1d5467e24b 2024-12-09T10:43:34,147 DEBUG [StoreCloser-TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/b434797b8ca5484eaea5cda03c94e994 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/b434797b8ca5484eaea5cda03c94e994 2024-12-09T10:43:34,148 DEBUG [StoreCloser-TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/dc507977e37e4a629ad46e4272127702 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/dc507977e37e4a629ad46e4272127702 2024-12-09T10:43:34,149 DEBUG [StoreCloser-TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/b545338a8d4f4120aede397c7c1e031c to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/b545338a8d4f4120aede397c7c1e031c 2024-12-09T10:43:34,150 DEBUG [StoreCloser-TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/a84f584c5da84ce4b9a391cf1945842e to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/a84f584c5da84ce4b9a391cf1945842e 2024-12-09T10:43:34,151 DEBUG [StoreCloser-TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/20843e37ceab4c24bbf55d56416a1859 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/20843e37ceab4c24bbf55d56416a1859 2024-12-09T10:43:34,152 DEBUG [StoreCloser-TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/5f241400dc1544e793ef75a324c16484 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/5f241400dc1544e793ef75a324c16484 2024-12-09T10:43:34,152 DEBUG [StoreCloser-TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/1dc8f315c46d488b96af735aa24366bb to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/1dc8f315c46d488b96af735aa24366bb 2024-12-09T10:43:34,153 DEBUG [StoreCloser-TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/f89cb34afa254ad7a2e33d0e8811986b to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/f89cb34afa254ad7a2e33d0e8811986b 2024-12-09T10:43:34,154 DEBUG [StoreCloser-TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/d780e2551af74785bc99d72f3d3ad316 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/d780e2551af74785bc99d72f3d3ad316 2024-12-09T10:43:34,154 DEBUG [StoreCloser-TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/34ad5b83260d47cc8c41094b52f5da65 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/34ad5b83260d47cc8c41094b52f5da65 2024-12-09T10:43:34,156 DEBUG [StoreCloser-TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/3a9d00eb495849c59b9168e4d88183d2, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/b5d7a7ca62f34f16908ba630a7886775, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/8862f157d1e349e09e21b281245e6783, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/572b6eea657540d0ad6c8541f603d2de, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/066f76192f194b41a89f3c2381b8c17d, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/59f7b5186ffe4c2dad6b664d6e1dc9b8, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/fc1dfdec982841ffb0f9d4b70e4fe590, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/faff8273cb8048ddbae1d87d05b35d96, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/3e655c7f6a1a4ba9905fd9dc5f4c664c, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/03c22604e71f453fa94b830a8878ac51, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/68aeed3e213d4d7bb39ddd3c053ba113, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/da7a8dd8545b45b48c7e1bb108ba80d2, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/86360001169a45a78d38183010f9cc23, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/d513904c790f4e4ab13f9c2682d147c6, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/079c6231155d4a169d394c30f629feac, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/b03be6c58be44294b35370243d809fdf, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/4ea5d3cd381f4031a25bdd6452226e1b, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/6026a918fa3043a8ae8419b1e7681138, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/92a9e2264aa44af7bdac85a71034a695, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/9c2140fb4625483ba65292e7345ddd29, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/0f57b19b90944c938b5b9ccc9c585649, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/fecc056859814a73ad2e61a1a69c04ed] to archive 2024-12-09T10:43:34,156 DEBUG [StoreCloser-TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T10:43:34,157 DEBUG [StoreCloser-TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/3a9d00eb495849c59b9168e4d88183d2 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/3a9d00eb495849c59b9168e4d88183d2 2024-12-09T10:43:34,158 DEBUG [StoreCloser-TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/b5d7a7ca62f34f16908ba630a7886775 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/b5d7a7ca62f34f16908ba630a7886775 2024-12-09T10:43:34,158 DEBUG [StoreCloser-TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/8862f157d1e349e09e21b281245e6783 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/8862f157d1e349e09e21b281245e6783 2024-12-09T10:43:34,159 DEBUG [StoreCloser-TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/572b6eea657540d0ad6c8541f603d2de to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/572b6eea657540d0ad6c8541f603d2de 2024-12-09T10:43:34,160 DEBUG [StoreCloser-TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/066f76192f194b41a89f3c2381b8c17d to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/066f76192f194b41a89f3c2381b8c17d 2024-12-09T10:43:34,161 DEBUG [StoreCloser-TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/59f7b5186ffe4c2dad6b664d6e1dc9b8 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/59f7b5186ffe4c2dad6b664d6e1dc9b8 2024-12-09T10:43:34,161 DEBUG [StoreCloser-TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/fc1dfdec982841ffb0f9d4b70e4fe590 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/fc1dfdec982841ffb0f9d4b70e4fe590 2024-12-09T10:43:34,162 DEBUG [StoreCloser-TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/faff8273cb8048ddbae1d87d05b35d96 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/faff8273cb8048ddbae1d87d05b35d96 2024-12-09T10:43:34,163 DEBUG [StoreCloser-TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/3e655c7f6a1a4ba9905fd9dc5f4c664c to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/3e655c7f6a1a4ba9905fd9dc5f4c664c 2024-12-09T10:43:34,163 DEBUG [StoreCloser-TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/03c22604e71f453fa94b830a8878ac51 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/03c22604e71f453fa94b830a8878ac51 2024-12-09T10:43:34,164 DEBUG [StoreCloser-TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/68aeed3e213d4d7bb39ddd3c053ba113 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/68aeed3e213d4d7bb39ddd3c053ba113 2024-12-09T10:43:34,165 DEBUG [StoreCloser-TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/da7a8dd8545b45b48c7e1bb108ba80d2 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/da7a8dd8545b45b48c7e1bb108ba80d2 2024-12-09T10:43:34,166 DEBUG [StoreCloser-TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/86360001169a45a78d38183010f9cc23 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/86360001169a45a78d38183010f9cc23 2024-12-09T10:43:34,166 DEBUG [StoreCloser-TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/d513904c790f4e4ab13f9c2682d147c6 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/d513904c790f4e4ab13f9c2682d147c6 2024-12-09T10:43:34,167 DEBUG [StoreCloser-TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/079c6231155d4a169d394c30f629feac to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/079c6231155d4a169d394c30f629feac 2024-12-09T10:43:34,168 DEBUG [StoreCloser-TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/b03be6c58be44294b35370243d809fdf to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/b03be6c58be44294b35370243d809fdf 2024-12-09T10:43:34,169 DEBUG [StoreCloser-TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/4ea5d3cd381f4031a25bdd6452226e1b to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/4ea5d3cd381f4031a25bdd6452226e1b 2024-12-09T10:43:34,170 DEBUG [StoreCloser-TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/6026a918fa3043a8ae8419b1e7681138 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/6026a918fa3043a8ae8419b1e7681138 2024-12-09T10:43:34,170 DEBUG [StoreCloser-TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/92a9e2264aa44af7bdac85a71034a695 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/92a9e2264aa44af7bdac85a71034a695 2024-12-09T10:43:34,171 DEBUG [StoreCloser-TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/9c2140fb4625483ba65292e7345ddd29 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/9c2140fb4625483ba65292e7345ddd29 2024-12-09T10:43:34,172 DEBUG [StoreCloser-TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/0f57b19b90944c938b5b9ccc9c585649 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/0f57b19b90944c938b5b9ccc9c585649 2024-12-09T10:43:34,173 DEBUG [StoreCloser-TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/fecc056859814a73ad2e61a1a69c04ed to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/fecc056859814a73ad2e61a1a69c04ed 2024-12-09T10:43:34,177 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/recovered.edits/423.seqid, newMaxSeqId=423, maxSeqId=4 2024-12-09T10:43:34,177 INFO [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. 2024-12-09T10:43:34,177 DEBUG [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1676): Region close journal for 92c0b293dd887d05ae6f705c47b9c951: Waiting for close lock at 1733741011482Running coprocessor pre-close hooks at 1733741011482Disabling compacts and flushes for region at 1733741011482Disabling writes for close at 1733741012471 (+989 ms)Obtaining lock to block concurrent updates at 1733741012471Preparing flush snapshotting stores in 92c0b293dd887d05ae6f705c47b9c951 at 1733741012471Finished memstore snapshotting TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1733741012471Flushing stores of TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951. at 1733741012472 (+1 ms)Flushing 92c0b293dd887d05ae6f705c47b9c951/A: creating writer at 1733741012472Flushing mob file 92c0b293dd887d05ae6f705c47b9c951/A: appending metadata at 1733741012475 (+3 ms)Flushing mob file 92c0b293dd887d05ae6f705c47b9c951/A: closing flushed file at 1733741012475Flushing 92c0b293dd887d05ae6f705c47b9c951/A: appending metadata at 1733741012882 (+407 ms)Flushing 92c0b293dd887d05ae6f705c47b9c951/A: closing flushed file at 1733741012882Flushing 92c0b293dd887d05ae6f705c47b9c951/B: creating writer at 1733741013289 (+407 ms)Flushing 92c0b293dd887d05ae6f705c47b9c951/B: appending metadata at 1733741013291 (+2 ms)Flushing 92c0b293dd887d05ae6f705c47b9c951/B: closing flushed file at 1733741013291Flushing 92c0b293dd887d05ae6f705c47b9c951/C: creating writer at 1733741013698 (+407 ms)Flushing 92c0b293dd887d05ae6f705c47b9c951/C: appending metadata at 1733741013700 (+2 ms)Flushing 92c0b293dd887d05ae6f705c47b9c951/C: closing flushed file at 1733741013700Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@418829b7: reopening flushed file at 1733741014107 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@acf9159: reopening flushed file at 1733741014110 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1b5e17b0: reopening flushed file at 1733741014112 (+2 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 92c0b293dd887d05ae6f705c47b9c951 in 1645ms, sequenceid=420, compaction requested=true at 1733741014116 (+4 ms)Writing region close event to WAL at 1733741014174 (+58 ms)Running coprocessor post-close hooks at 1733741014177 (+3 ms)Closed at 1733741014177 2024-12-09T10:43:34,179 INFO [RS_CLOSE_REGION-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] handler.UnassignRegionHandler(157): Closed 92c0b293dd887d05ae6f705c47b9c951 2024-12-09T10:43:34,179 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=166 updating hbase:meta row=92c0b293dd887d05ae6f705c47b9c951, regionState=CLOSED 2024-12-09T10:43:34,181 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=167, ppid=166, state=RUNNABLE, hasLock=false; CloseRegionProcedure 92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 because future has completed 2024-12-09T10:43:34,182 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-12-09T10:43:34,182 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; CloseRegionProcedure 92c0b293dd887d05ae6f705c47b9c951, server=7ea43b39fc24,36101,1733740827678 in 2.8510 sec 2024-12-09T10:43:34,183 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=166, resume processing ppid=165 2024-12-09T10:43:34,183 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, ppid=165, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=92c0b293dd887d05ae6f705c47b9c951, UNASSIGN in 2.8540 sec 2024-12-09T10:43:34,185 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-12-09T10:43:34,185 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.8570 sec 2024-12-09T10:43:34,186 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733741014186"}]},"ts":"1733741014186"} 2024-12-09T10:43:34,187 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-09T10:43:34,187 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-09T10:43:34,188 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.8650 sec 2024-12-09T10:43:35,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-09T10:43:35,480 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-09T10:43:35,480 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-12-09T10:43:35,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] procedure2.ProcedureExecutor(1139): Stored pid=168, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T10:43:35,482 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=168, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T10:43:35,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-12-09T10:43:35,482 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=168, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T10:43:35,484 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951 2024-12-09T10:43:35,486 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A, FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B, FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C, FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/recovered.edits] 2024-12-09T10:43:35,488 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/31518a620bbe47a5b820c9579efddcb3 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/31518a620bbe47a5b820c9579efddcb3 2024-12-09T10:43:35,488 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/633e2eaa88a341d881f1ae2268f00fee to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/633e2eaa88a341d881f1ae2268f00fee 2024-12-09T10:43:35,489 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/b0272bc4d07e4c6aa8d46b7b9cf377ea to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/A/b0272bc4d07e4c6aa8d46b7b9cf377ea 2024-12-09T10:43:35,491 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/33b60c18dea044fba1620a0cbb22a45b to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/33b60c18dea044fba1620a0cbb22a45b 2024-12-09T10:43:35,491 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/75bc29876a944a188a3b159a6518ac21 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/75bc29876a944a188a3b159a6518ac21 2024-12-09T10:43:35,492 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/fa706f5fcb584566993e2171df311645 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/B/fa706f5fcb584566993e2171df311645 2024-12-09T10:43:35,494 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/3a26200acebe48b387989e00cf5cc259 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/3a26200acebe48b387989e00cf5cc259 2024-12-09T10:43:35,494 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/b524cccac9584583944c2182ac33bb12 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/b524cccac9584583944c2182ac33bb12 2024-12-09T10:43:35,495 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/b7b91faae7ad48fcafee25b5cfee82a0 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/C/b7b91faae7ad48fcafee25b5cfee82a0 2024-12-09T10:43:35,497 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/recovered.edits/423.seqid to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951/recovered.edits/423.seqid 2024-12-09T10:43:35,498 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/default/TestAcidGuarantees/92c0b293dd887d05ae6f705c47b9c951 2024-12-09T10:43:35,498 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-09T10:43:35,498 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-09T10:43:35,498 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-09T10:43:35,500 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412090ee1f404e3e84508bec2d6c36ffc9286_92c0b293dd887d05ae6f705c47b9c951 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412090ee1f404e3e84508bec2d6c36ffc9286_92c0b293dd887d05ae6f705c47b9c951 2024-12-09T10:43:35,501 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412093e0945c50c9b4ded848e47559ce59289_92c0b293dd887d05ae6f705c47b9c951 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412093e0945c50c9b4ded848e47559ce59289_92c0b293dd887d05ae6f705c47b9c951 2024-12-09T10:43:35,502 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120944cbf5dc179c4a518be0872fe9993406_92c0b293dd887d05ae6f705c47b9c951 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120944cbf5dc179c4a518be0872fe9993406_92c0b293dd887d05ae6f705c47b9c951 2024-12-09T10:43:35,502 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412095bcdf44aa36a451aa7ffefff561b0286_92c0b293dd887d05ae6f705c47b9c951 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412095bcdf44aa36a451aa7ffefff561b0286_92c0b293dd887d05ae6f705c47b9c951 2024-12-09T10:43:35,503 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412095d3d6025ae48437795564a3ffcc1f306_92c0b293dd887d05ae6f705c47b9c951 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412095d3d6025ae48437795564a3ffcc1f306_92c0b293dd887d05ae6f705c47b9c951 2024-12-09T10:43:35,504 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209715d91b173ef41c8a5b7e7fca008514b_92c0b293dd887d05ae6f705c47b9c951 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209715d91b173ef41c8a5b7e7fca008514b_92c0b293dd887d05ae6f705c47b9c951 2024-12-09T10:43:35,504 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209862d65847d0f414ba3ceec3729fbe1a9_92c0b293dd887d05ae6f705c47b9c951 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209862d65847d0f414ba3ceec3729fbe1a9_92c0b293dd887d05ae6f705c47b9c951 2024-12-09T10:43:35,505 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412098cc62f0a7f5d49b6afacbb74908bbd74_92c0b293dd887d05ae6f705c47b9c951 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412098cc62f0a7f5d49b6afacbb74908bbd74_92c0b293dd887d05ae6f705c47b9c951 2024-12-09T10:43:35,506 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120993ca667866f642a6b6e2441f08759c36_92c0b293dd887d05ae6f705c47b9c951 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120993ca667866f642a6b6e2441f08759c36_92c0b293dd887d05ae6f705c47b9c951 2024-12-09T10:43:35,507 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209959ed986e3f240ba8eb0eb5ed8fe08f2_92c0b293dd887d05ae6f705c47b9c951 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209959ed986e3f240ba8eb0eb5ed8fe08f2_92c0b293dd887d05ae6f705c47b9c951 2024-12-09T10:43:35,507 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209ca2bdb79413d4d80a61e055ed74a01e8_92c0b293dd887d05ae6f705c47b9c951 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209ca2bdb79413d4d80a61e055ed74a01e8_92c0b293dd887d05ae6f705c47b9c951 2024-12-09T10:43:35,508 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209cc4070a6f88747e980776e6c8ddb5eff_92c0b293dd887d05ae6f705c47b9c951 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209cc4070a6f88747e980776e6c8ddb5eff_92c0b293dd887d05ae6f705c47b9c951 2024-12-09T10:43:35,509 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209d3fe4b6375d647c79741cafd22e2ed9b_92c0b293dd887d05ae6f705c47b9c951 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209d3fe4b6375d647c79741cafd22e2ed9b_92c0b293dd887d05ae6f705c47b9c951 2024-12-09T10:43:35,510 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209d6326909be424b6e8c89eb2030e5ed63_92c0b293dd887d05ae6f705c47b9c951 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209d6326909be424b6e8c89eb2030e5ed63_92c0b293dd887d05ae6f705c47b9c951 2024-12-09T10:43:35,511 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209d8f3af9deda34709a0cfa20ec084a876_92c0b293dd887d05ae6f705c47b9c951 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209d8f3af9deda34709a0cfa20ec084a876_92c0b293dd887d05ae6f705c47b9c951 2024-12-09T10:43:35,512 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209f2b0d84b48a944bd8ab4eb66a9ea8c70_92c0b293dd887d05ae6f705c47b9c951 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209f2b0d84b48a944bd8ab4eb66a9ea8c70_92c0b293dd887d05ae6f705c47b9c951 2024-12-09T10:43:35,512 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209f6fa4996e7e745e093b59e8ba959e5de_92c0b293dd887d05ae6f705c47b9c951 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209f6fa4996e7e745e093b59e8ba959e5de_92c0b293dd887d05ae6f705c47b9c951 2024-12-09T10:43:35,513 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209f899c477f016441d963248af158a6dcc_92c0b293dd887d05ae6f705c47b9c951 to hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209f899c477f016441d963248af158a6dcc_92c0b293dd887d05ae6f705c47b9c951 2024-12-09T10:43:35,513 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-09T10:43:35,515 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=168, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T10:43:35,518 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-09T10:43:35,519 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-09T10:43:35,520 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=168, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T10:43:35,520 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-09T10:43:35,520 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733741015520"}]},"ts":"9223372036854775807"} 2024-12-09T10:43:35,522 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-09T10:43:35,522 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 92c0b293dd887d05ae6f705c47b9c951, NAME => 'TestAcidGuarantees,,1733740986220.92c0b293dd887d05ae6f705c47b9c951.', STARTKEY => '', ENDKEY => ''}] 2024-12-09T10:43:35,522 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-09T10:43:35,522 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733741015522"}]},"ts":"9223372036854775807"} 2024-12-09T10:43:35,524 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-09T10:43:35,524 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=168, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T10:43:35,525 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 44 msec 2024-12-09T10:43:35,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-12-09T10:43:35,590 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-09T10:43:35,590 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-09T10:43:35,600 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=251 (was 248) - Thread LEAK? -, OpenFileDescriptor=457 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=371 (was 317) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7230 (was 7301) 2024-12-09T10:43:35,600 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-09T10:43:35,600 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-09T10:43:35,600 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T10:43:35,600 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:43:35,600 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:43:35,600 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T10:43:35,600 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-09T10:43:35,600 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1432429954, stopped=false 2024-12-09T10:43:35,601 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=7ea43b39fc24,35095,1733740826918 2024-12-09T10:43:35,603 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35095-0x100bd55aca10000, quorum=127.0.0.1:50851, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-09T10:43:35,603 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36101-0x100bd55aca10001, quorum=127.0.0.1:50851, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-09T10:43:35,603 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36101-0x100bd55aca10001, quorum=127.0.0.1:50851, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T10:43:35,603 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35095-0x100bd55aca10000, quorum=127.0.0.1:50851, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T10:43:35,603 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-09T10:43:35,604 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-09T10:43:35,604 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T10:43:35,604 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:43:35,604 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:35095-0x100bd55aca10000, quorum=127.0.0.1:50851, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T10:43:35,604 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36101-0x100bd55aca10001, quorum=127.0.0.1:50851, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T10:43:35,604 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '7ea43b39fc24,36101,1733740827678' ***** 2024-12-09T10:43:35,604 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-09T10:43:35,605 INFO [RS:0;7ea43b39fc24:36101 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-09T10:43:35,605 INFO [RS:0;7ea43b39fc24:36101 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-09T10:43:35,605 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-09T10:43:35,605 INFO [RS:0;7ea43b39fc24:36101 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-09T10:43:35,605 INFO [RS:0;7ea43b39fc24:36101 {}] regionserver.HRegionServer(959): stopping server 7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:35,605 INFO [RS:0;7ea43b39fc24:36101 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-09T10:43:35,605 INFO [RS:0;7ea43b39fc24:36101 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;7ea43b39fc24:36101. 2024-12-09T10:43:35,605 DEBUG [RS:0;7ea43b39fc24:36101 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T10:43:35,605 DEBUG [RS:0;7ea43b39fc24:36101 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:43:35,605 INFO [RS:0;7ea43b39fc24:36101 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-09T10:43:35,605 INFO [RS:0;7ea43b39fc24:36101 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-09T10:43:35,605 INFO [RS:0;7ea43b39fc24:36101 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-09T10:43:35,606 INFO [RS:0;7ea43b39fc24:36101 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-09T10:43:35,606 INFO [RS:0;7ea43b39fc24:36101 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-09T10:43:35,606 DEBUG [RS:0;7ea43b39fc24:36101 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-12-09T10:43:35,607 DEBUG [RS_CLOSE_META-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-09T10:43:35,607 INFO [RS_CLOSE_META-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-09T10:43:35,607 DEBUG [RS:0;7ea43b39fc24:36101 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-09T10:43:35,607 DEBUG [RS_CLOSE_META-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-09T10:43:35,607 DEBUG [RS_CLOSE_META-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-09T10:43:35,607 DEBUG [RS_CLOSE_META-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-09T10:43:35,607 INFO [RS_CLOSE_META-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-12-09T10:43:35,632 DEBUG [RS_CLOSE_META-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/hbase/meta/1588230740/.tmp/info/dfc4b09762fb4db8a2fdfcca29aacaf4 is 95, key is TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5./info:/1733740858342/DeleteFamily/seqid=0 2024-12-09T10:43:35,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742434_1610 (size=6514) 2024-12-09T10:43:35,701 INFO [regionserver/7ea43b39fc24:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-09T10:43:35,807 DEBUG [RS:0;7ea43b39fc24:36101 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-09T10:43:36,007 DEBUG [RS:0;7ea43b39fc24:36101 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-09T10:43:36,036 INFO [RS_CLOSE_META-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/hbase/meta/1588230740/.tmp/info/dfc4b09762fb4db8a2fdfcca29aacaf4 2024-12-09T10:43:36,054 DEBUG [RS_CLOSE_META-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/hbase/meta/1588230740/.tmp/ns/cf19412527f347c981ccfbd97dac71b6 is 93, key is TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5./ns:/1733740858290/DeleteFamily/seqid=0 2024-12-09T10:43:36,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742435_1611 (size=5906) 2024-12-09T10:43:36,207 DEBUG [RS:0;7ea43b39fc24:36101 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-09T10:43:36,210 INFO [regionserver/7ea43b39fc24:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-09T10:43:36,210 INFO [regionserver/7ea43b39fc24:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-09T10:43:36,408 DEBUG [RS:0;7ea43b39fc24:36101 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-09T10:43:36,457 INFO [RS_CLOSE_META-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/hbase/meta/1588230740/.tmp/ns/cf19412527f347c981ccfbd97dac71b6 2024-12-09T10:43:36,474 DEBUG [RS_CLOSE_META-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/hbase/meta/1588230740/.tmp/rep_barrier/5fd9ecc3861a4fdbac9b5f472a0eb790 is 102, key is TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5./rep_barrier:/1733740858290/DeleteFamily/seqid=0 2024-12-09T10:43:36,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742436_1612 (size=6025) 2024-12-09T10:43:36,608 DEBUG [RS:0;7ea43b39fc24:36101 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-09T10:43:36,808 DEBUG [RS:0;7ea43b39fc24:36101 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-09T10:43:36,878 INFO [RS_CLOSE_META-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/hbase/meta/1588230740/.tmp/rep_barrier/5fd9ecc3861a4fdbac9b5f472a0eb790 2024-12-09T10:43:36,895 DEBUG [RS_CLOSE_META-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/hbase/meta/1588230740/.tmp/table/1b1c59194a2f4056bcb4dca864ef69c6 is 96, key is TestAcidGuarantees,,1733740831730.1d9d6bba7dcd09663c7b08474fc1c1b5./table:/1733740858290/DeleteFamily/seqid=0 2024-12-09T10:43:36,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742437_1613 (size=5936) 2024-12-09T10:43:37,008 DEBUG [RS:0;7ea43b39fc24:36101 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-09T10:43:37,208 DEBUG [RS:0;7ea43b39fc24:36101 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-09T10:43:37,298 INFO [RS_CLOSE_META-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/hbase/meta/1588230740/.tmp/table/1b1c59194a2f4056bcb4dca864ef69c6 2024-12-09T10:43:37,302 DEBUG [RS_CLOSE_META-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/hbase/meta/1588230740/.tmp/info/dfc4b09762fb4db8a2fdfcca29aacaf4 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/hbase/meta/1588230740/info/dfc4b09762fb4db8a2fdfcca29aacaf4 2024-12-09T10:43:37,304 INFO [RS_CLOSE_META-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/hbase/meta/1588230740/info/dfc4b09762fb4db8a2fdfcca29aacaf4, entries=12, sequenceid=90, filesize=6.4 K 2024-12-09T10:43:37,305 DEBUG [RS_CLOSE_META-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/hbase/meta/1588230740/.tmp/ns/cf19412527f347c981ccfbd97dac71b6 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/hbase/meta/1588230740/ns/cf19412527f347c981ccfbd97dac71b6 2024-12-09T10:43:37,307 INFO [RS_CLOSE_META-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/hbase/meta/1588230740/ns/cf19412527f347c981ccfbd97dac71b6, entries=8, sequenceid=90, filesize=5.8 K 2024-12-09T10:43:37,308 DEBUG [RS_CLOSE_META-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/hbase/meta/1588230740/.tmp/rep_barrier/5fd9ecc3861a4fdbac9b5f472a0eb790 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/hbase/meta/1588230740/rep_barrier/5fd9ecc3861a4fdbac9b5f472a0eb790 2024-12-09T10:43:37,310 INFO [RS_CLOSE_META-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/hbase/meta/1588230740/rep_barrier/5fd9ecc3861a4fdbac9b5f472a0eb790, entries=6, sequenceid=90, filesize=5.9 K 2024-12-09T10:43:37,311 DEBUG [RS_CLOSE_META-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/hbase/meta/1588230740/.tmp/table/1b1c59194a2f4056bcb4dca864ef69c6 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/hbase/meta/1588230740/table/1b1c59194a2f4056bcb4dca864ef69c6 2024-12-09T10:43:37,313 INFO [RS_CLOSE_META-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/hbase/meta/1588230740/table/1b1c59194a2f4056bcb4dca864ef69c6, entries=7, sequenceid=90, filesize=5.8 K 2024-12-09T10:43:37,314 INFO [RS_CLOSE_META-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1707ms, sequenceid=90, compaction requested=false 2024-12-09T10:43:37,317 DEBUG [RS_CLOSE_META-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-12-09T10:43:37,318 DEBUG [RS_CLOSE_META-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-09T10:43:37,318 INFO [RS_CLOSE_META-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-09T10:43:37,318 DEBUG [RS_CLOSE_META-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733741015606Running coprocessor pre-close hooks at 1733741015607 (+1 ms)Disabling compacts and flushes for region at 1733741015607Disabling writes for close at 1733741015607Obtaining lock to block concurrent updates at 1733741015607Preparing flush snapshotting stores in 1588230740 at 1733741015607Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1733741015608 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733741015608Flushing 1588230740/info: creating writer at 1733741015608Flushing 1588230740/info: appending metadata at 1733741015631 (+23 ms)Flushing 1588230740/info: closing flushed file at 1733741015631Flushing 1588230740/ns: creating writer at 1733741016039 (+408 ms)Flushing 1588230740/ns: appending metadata at 1733741016053 (+14 ms)Flushing 1588230740/ns: closing flushed file at 1733741016053Flushing 1588230740/rep_barrier: creating writer at 1733741016460 (+407 ms)Flushing 1588230740/rep_barrier: appending metadata at 1733741016474 (+14 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1733741016474Flushing 1588230740/table: creating writer at 1733741016881 (+407 ms)Flushing 1588230740/table: appending metadata at 1733741016895 (+14 ms)Flushing 1588230740/table: closing flushed file at 1733741016895Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4abe2581: reopening flushed file at 1733741017301 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3ad1562c: reopening flushed file at 1733741017304 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@9ce6d22: reopening flushed file at 1733741017307 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5916e12a: reopening flushed file at 1733741017310 (+3 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1707ms, sequenceid=90, compaction requested=false at 1733741017314 (+4 ms)Writing region close event to WAL at 1733741017315 (+1 ms)Running coprocessor post-close hooks at 1733741017318 (+3 ms)Closed at 1733741017318 2024-12-09T10:43:37,318 DEBUG [RS_CLOSE_META-regionserver/7ea43b39fc24:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-09T10:43:37,343 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-09T10:43:37,343 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-09T10:43:37,344 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-09T10:43:37,409 INFO [RS:0;7ea43b39fc24:36101 {}] regionserver.HRegionServer(976): stopping server 7ea43b39fc24,36101,1733740827678; all regions closed. 2024-12-09T10:43:37,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741834_1010 (size=25344) 2024-12-09T10:43:37,415 DEBUG [RS:0;7ea43b39fc24:36101 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/oldWALs 2024-12-09T10:43:37,415 INFO [RS:0;7ea43b39fc24:36101 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 7ea43b39fc24%2C36101%2C1733740827678.meta:.meta(num 1733740831265) 2024-12-09T10:43:37,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741832_1008 (size=18009844) 2024-12-09T10:43:37,419 DEBUG [RS:0;7ea43b39fc24:36101 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/oldWALs 2024-12-09T10:43:37,419 INFO [RS:0;7ea43b39fc24:36101 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 7ea43b39fc24%2C36101%2C1733740827678:(num 1733740830371) 2024-12-09T10:43:37,419 DEBUG [RS:0;7ea43b39fc24:36101 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T10:43:37,419 INFO [RS:0;7ea43b39fc24:36101 {}] regionserver.LeaseManager(133): Closed leases 2024-12-09T10:43:37,419 INFO [RS:0;7ea43b39fc24:36101 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-09T10:43:37,419 INFO [RS:0;7ea43b39fc24:36101 {}] hbase.ChoreService(370): Chore service for: regionserver/7ea43b39fc24:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-09T10:43:37,419 INFO [RS:0;7ea43b39fc24:36101 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-09T10:43:37,419 INFO [regionserver/7ea43b39fc24:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-09T10:43:37,420 INFO [RS:0;7ea43b39fc24:36101 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:36101 2024-12-09T10:43:37,424 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36101-0x100bd55aca10001, quorum=127.0.0.1:50851, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/7ea43b39fc24,36101,1733740827678 2024-12-09T10:43:37,424 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35095-0x100bd55aca10000, quorum=127.0.0.1:50851, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-09T10:43:37,424 INFO [RS:0;7ea43b39fc24:36101 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-09T10:43:37,426 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [7ea43b39fc24,36101,1733740827678] 2024-12-09T10:43:37,427 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/7ea43b39fc24,36101,1733740827678 already deleted, retry=false 2024-12-09T10:43:37,427 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 7ea43b39fc24,36101,1733740827678 expired; onlineServers=0 2024-12-09T10:43:37,427 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '7ea43b39fc24,35095,1733740826918' ***** 2024-12-09T10:43:37,428 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-09T10:43:37,428 INFO [M:0;7ea43b39fc24:35095 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-09T10:43:37,428 INFO [M:0;7ea43b39fc24:35095 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-09T10:43:37,428 DEBUG [M:0;7ea43b39fc24:35095 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-09T10:43:37,428 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-09T10:43:37,428 DEBUG [M:0;7ea43b39fc24:35095 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-09T10:43:37,428 DEBUG [master/7ea43b39fc24:0:becomeActiveMaster-HFileCleaner.large.0-1733740830010 {}] cleaner.HFileCleaner(306): Exit Thread[master/7ea43b39fc24:0:becomeActiveMaster-HFileCleaner.large.0-1733740830010,5,FailOnTimeoutGroup] 2024-12-09T10:43:37,428 DEBUG [master/7ea43b39fc24:0:becomeActiveMaster-HFileCleaner.small.0-1733740830011 {}] cleaner.HFileCleaner(306): Exit Thread[master/7ea43b39fc24:0:becomeActiveMaster-HFileCleaner.small.0-1733740830011,5,FailOnTimeoutGroup] 2024-12-09T10:43:37,428 INFO [M:0;7ea43b39fc24:35095 {}] hbase.ChoreService(370): Chore service for: master/7ea43b39fc24:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-09T10:43:37,428 INFO [M:0;7ea43b39fc24:35095 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-09T10:43:37,428 DEBUG [M:0;7ea43b39fc24:35095 {}] master.HMaster(1795): Stopping service threads 2024-12-09T10:43:37,428 INFO [M:0;7ea43b39fc24:35095 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-09T10:43:37,428 INFO [M:0;7ea43b39fc24:35095 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-09T10:43:37,429 ERROR [M:0;7ea43b39fc24:35095 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-12-09T10:43:37,429 INFO [M:0;7ea43b39fc24:35095 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-09T10:43:37,429 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-09T10:43:37,430 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35095-0x100bd55aca10000, quorum=127.0.0.1:50851, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-09T10:43:37,430 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35095-0x100bd55aca10000, quorum=127.0.0.1:50851, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T10:43:37,430 DEBUG [M:0;7ea43b39fc24:35095 {}] zookeeper.ZKUtil(347): master:35095-0x100bd55aca10000, quorum=127.0.0.1:50851, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-09T10:43:37,430 WARN [M:0;7ea43b39fc24:35095 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-09T10:43:37,431 INFO [M:0;7ea43b39fc24:35095 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/.lastflushedseqids 2024-12-09T10:43:37,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742438_1614 (size=72) 2024-12-09T10:43:37,526 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36101-0x100bd55aca10001, quorum=127.0.0.1:50851, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:43:37,526 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36101-0x100bd55aca10001, quorum=127.0.0.1:50851, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:43:37,527 INFO [RS:0;7ea43b39fc24:36101 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-09T10:43:37,527 INFO [RS:0;7ea43b39fc24:36101 {}] regionserver.HRegionServer(1031): Exiting; stopping=7ea43b39fc24,36101,1733740827678; zookeeper connection closed. 2024-12-09T10:43:37,527 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@3d83e1d7 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@3d83e1d7 2024-12-09T10:43:37,528 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-12-09T10:43:37,840 INFO [M:0;7ea43b39fc24:35095 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-09T10:43:37,840 INFO [M:0;7ea43b39fc24:35095 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-09T10:43:37,840 DEBUG [M:0;7ea43b39fc24:35095 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-09T10:43:37,840 INFO [M:0;7ea43b39fc24:35095 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T10:43:37,840 DEBUG [M:0;7ea43b39fc24:35095 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T10:43:37,840 DEBUG [M:0;7ea43b39fc24:35095 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-09T10:43:37,840 DEBUG [M:0;7ea43b39fc24:35095 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T10:43:37,840 INFO [M:0;7ea43b39fc24:35095 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=762.17 KB heapSize=932.80 KB 2024-12-09T10:43:37,856 DEBUG [M:0;7ea43b39fc24:35095 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/68c3b26f3ce246a1af594bae07325bd8 is 82, key is hbase:meta,,1/info:regioninfo/1733740831351/Put/seqid=0 2024-12-09T10:43:37,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742439_1615 (size=5672) 2024-12-09T10:43:38,259 INFO [M:0;7ea43b39fc24:35095 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2105 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/68c3b26f3ce246a1af594bae07325bd8 2024-12-09T10:43:38,278 DEBUG [M:0;7ea43b39fc24:35095 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/9618180dec2e45c7abcfe4cbe72fbae5 is 2479, key is \x00\x00\x00\x00\x00\x00\x00X/proc:d/1733740932210/Put/seqid=0 2024-12-09T10:43:38,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742440_1616 (size=44998) 2024-12-09T10:43:38,682 INFO [M:0;7ea43b39fc24:35095 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=761.62 KB at sequenceid=2105 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/9618180dec2e45c7abcfe4cbe72fbae5 2024-12-09T10:43:38,686 INFO [M:0;7ea43b39fc24:35095 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 9618180dec2e45c7abcfe4cbe72fbae5 2024-12-09T10:43:38,701 DEBUG [M:0;7ea43b39fc24:35095 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/784cce9e1fb546338a346a52de469c91 is 69, key is 7ea43b39fc24,36101,1733740827678/rs:state/1733740830131/Put/seqid=0 2024-12-09T10:43:38,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073742441_1617 (size=5156) 2024-12-09T10:43:39,104 INFO [M:0;7ea43b39fc24:35095 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2105 (bloomFilter=true), to=hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/784cce9e1fb546338a346a52de469c91 2024-12-09T10:43:39,108 DEBUG [M:0;7ea43b39fc24:35095 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/68c3b26f3ce246a1af594bae07325bd8 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/68c3b26f3ce246a1af594bae07325bd8 2024-12-09T10:43:39,111 INFO [M:0;7ea43b39fc24:35095 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/68c3b26f3ce246a1af594bae07325bd8, entries=8, sequenceid=2105, filesize=5.5 K 2024-12-09T10:43:39,111 DEBUG [M:0;7ea43b39fc24:35095 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/9618180dec2e45c7abcfe4cbe72fbae5 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/9618180dec2e45c7abcfe4cbe72fbae5 2024-12-09T10:43:39,114 INFO [M:0;7ea43b39fc24:35095 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 9618180dec2e45c7abcfe4cbe72fbae5 2024-12-09T10:43:39,114 INFO [M:0;7ea43b39fc24:35095 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/9618180dec2e45c7abcfe4cbe72fbae5, entries=168, sequenceid=2105, filesize=43.9 K 2024-12-09T10:43:39,114 DEBUG [M:0;7ea43b39fc24:35095 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/784cce9e1fb546338a346a52de469c91 as hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/784cce9e1fb546338a346a52de469c91 2024-12-09T10:43:39,116 INFO [M:0;7ea43b39fc24:35095 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34553/user/jenkins/test-data/2b514445-1e4c-5954-120c-80e40d3a1160/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/784cce9e1fb546338a346a52de469c91, entries=1, sequenceid=2105, filesize=5.0 K 2024-12-09T10:43:39,117 INFO [M:0;7ea43b39fc24:35095 {}] regionserver.HRegion(3140): Finished flush of dataSize ~762.17 KB/780464, heapSize ~932.51 KB/954888, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1277ms, sequenceid=2105, compaction requested=false 2024-12-09T10:43:39,118 INFO [M:0;7ea43b39fc24:35095 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T10:43:39,119 DEBUG [M:0;7ea43b39fc24:35095 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733741017840Disabling compacts and flushes for region at 1733741017840Disabling writes for close at 1733741017840Obtaining lock to block concurrent updates at 1733741017841 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733741017841Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=780464, getHeapSize=955128, getOffHeapSize=0, getCellsCount=2298 at 1733741017841Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733741017841Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733741017841Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733741017855 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733741017855Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733741018262 (+407 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733741018278 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733741018278Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733741018686 (+408 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733741018700 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733741018700Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1207648b: reopening flushed file at 1733741019108 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@58638507: reopening flushed file at 1733741019111 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7149db32: reopening flushed file at 1733741019114 (+3 ms)Finished flush of dataSize ~762.17 KB/780464, heapSize ~932.51 KB/954888, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1277ms, sequenceid=2105, compaction requested=false at 1733741019117 (+3 ms)Writing region close event to WAL at 1733741019118 (+1 ms)Closed at 1733741019118 2024-12-09T10:43:39,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741830_1006 (size=918480) 2024-12-09T10:43:39,122 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-09T10:43:39,122 INFO [M:0;7ea43b39fc24:35095 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-09T10:43:39,122 INFO [M:0;7ea43b39fc24:35095 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:35095 2024-12-09T10:43:39,122 INFO [M:0;7ea43b39fc24:35095 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-09T10:43:39,224 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35095-0x100bd55aca10000, quorum=127.0.0.1:50851, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:43:39,224 INFO [M:0;7ea43b39fc24:35095 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-09T10:43:39,224 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35095-0x100bd55aca10000, quorum=127.0.0.1:50851, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T10:43:39,229 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3615128e{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-09T10:43:39,232 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@580c1d20{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-09T10:43:39,232 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-09T10:43:39,232 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@67a8203b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-09T10:43:39,232 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@603a60a2{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/376c0fb7-94bc-49c9-3c3c-f09e60d8d2eb/hadoop.log.dir/,STOPPED} 2024-12-09T10:43:39,235 WARN [BP-1165291760-172.17.0.3-1733740823632 heartbeating to localhost/127.0.0.1:34553 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-09T10:43:39,235 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-09T10:43:39,235 WARN [BP-1165291760-172.17.0.3-1733740823632 heartbeating to localhost/127.0.0.1:34553 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1165291760-172.17.0.3-1733740823632 (Datanode Uuid db470eaf-d669-4d74-9769-0b3e76661fc5) service to localhost/127.0.0.1:34553 2024-12-09T10:43:39,235 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-09T10:43:39,237 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/376c0fb7-94bc-49c9-3c3c-f09e60d8d2eb/cluster_8fa5eac4-e45c-afe9-0851-24d1455445f8/data/data1/current/BP-1165291760-172.17.0.3-1733740823632 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-09T10:43:39,237 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/376c0fb7-94bc-49c9-3c3c-f09e60d8d2eb/cluster_8fa5eac4-e45c-afe9-0851-24d1455445f8/data/data2/current/BP-1165291760-172.17.0.3-1733740823632 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-09T10:43:39,238 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-09T10:43:39,244 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@55a6f9c7{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-09T10:43:39,245 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@40742525{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-09T10:43:39,245 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-09T10:43:39,245 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6efc0ba4{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-09T10:43:39,245 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4516de21{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/376c0fb7-94bc-49c9-3c3c-f09e60d8d2eb/hadoop.log.dir/,STOPPED} 2024-12-09T10:43:39,260 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-09T10:43:39,376 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down